Implement the launcher

The commit implements the launcher for tag retention

Signed-off-by: Wenkai Yin <yinw@vmware.com>

src/core/api/chart_repository.go

@@ -46,6 +46,11 @@ const (
 // chartController is a singleton instance
 var chartController *chartserver.Controller
 
+// GetChartController returns the chart controller
+func GetChartController() *chartserver.Controller {
+	return chartController
+}
+
 // ChartRepositoryAPI provides related API handlers for the chart repository APIs
 type ChartRepositoryAPI struct {
 	// The base controller to provide common utilities

src/pkg/project/manager.go

@@ -0,0 +1,66 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package project
+
+import (
+	"fmt"
+
+	"github.com/goharbor/harbor/src/common/dao"
+	"github.com/goharbor/harbor/src/common/models"
+)
+
+var (
+	// Mgr is an instance of the default project Manager
+	Mgr = New()
+)
+
+// Manager is used for project management
+// currently, the interface only defines the methods needed for tag retention
+// will expand it when doing refactor
+type Manager interface {
+	// List projects according to the query
+	List(...*models.ProjectQueryParam) ([]*models.Project, error)
+	// Get the project specified by the ID or name
+	Get(interface{}) (*models.Project, error)
+}
+
+// New returns a default implementation of Manager
+func New() Manager {
+	return &manager{}
+}
+
+type manager struct{}
+
+// List projects according to the query
+func (m *manager) List(query ...*models.ProjectQueryParam) ([]*models.Project, error) {
+	var q *models.ProjectQueryParam
+	if len(query) > 0 {
+		q = query[0]
+	}
+	return dao.GetProjects(q)
+}
+
+// Get the project specified by the ID
+func (m *manager) Get(idOrName interface{}) (*models.Project, error) {
+	id, ok := idOrName.(int64)
+	if ok {
+		return dao.GetProjectByID(id)
+	}
+	name, ok := idOrName.(string)
+	if ok {
+		return dao.GetProjectByName(name)
+	}
+	return nil, fmt.Errorf("invalid parameter: %v, should be ID(int64) or name(string)", idOrName)
+}

src/pkg/repository/manager.go

@@ -0,0 +1,61 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package repository
+
+import (
+	"github.com/goharbor/harbor/src/chartserver"
+	"github.com/goharbor/harbor/src/common/dao"
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/core/api"
+	"github.com/goharbor/harbor/src/pkg/project"
+)
+
+var (
+	// Mgr is an instance of the default repository Manager
+	Mgr = New()
+)
+
+// Manager is used for repository management
+// currently, the interface only defines the methods needed for tag retention
+// will expand it when doing refactor
+type Manager interface {
+	// List image repositories under the project specified by the ID
+	ListImageRepositories(projectID int64) ([]*models.RepoRecord, error)
+	// List chart repositories under the project specified by the ID
+	ListChartRepositories(projectID int64) ([]*chartserver.ChartInfo, error)
+}
+
+// New returns a default implementation of Manager
+func New() Manager {
+	return &manager{}
+}
+
+type manager struct{}
+
+// List image repositories under the project specified by the ID
+func (m *manager) ListImageRepositories(projectID int64) ([]*models.RepoRecord, error) {
+	return dao.GetRepositories(&models.RepositoryQuery{
+		ProjectIDs: []int64{projectID},
+	})
+}
+
+// List chart repositories under the project specified by the ID
+func (m *manager) ListChartRepositories(projectID int64) ([]*chartserver.ChartInfo, error) {
+	project, err := project.Mgr.Get(projectID)
+	if err != nil {
+		return nil, err
+	}
+	return api.GetChartController().ListCharts(project.Name)
+}

src/pkg/retention/client.go

@@ -14,7 +14,10 @@
 
 package retention
 
-import "github.com/goharbor/harbor/src/pkg/retention/res"
+import (
+	"github.com/goharbor/harbor/src/pkg/retention/policy"
+	"github.com/goharbor/harbor/src/pkg/retention/res"
+)
 
 // Client is designed to access core service to get required infos
 type Client interface {
@@ -36,6 +39,17 @@ type Client interface {
 	//  Returns:
 	//    error : common error if any errors occurred
 	Delete(candidate *res.Candidate) error
+
+	// SubmitTask to jobservice
+	//
+	//  Arguments:
+	//    repository: *res.Repository : repository info
+	//    meta *policy.LiteMeta       : policy lite metadata
+	//
+	//  Returns:
+	//    string : the job ID
+	//    error  : common error if any errors occurred
+	SubmitTask(repository *res.Repository, meta *policy.LiteMeta) (string, error)
 }
 
 // New basic client
@@ -57,3 +71,8 @@ func (bc *basicClient) GetCandidates(repo *res.Repository) ([]*res.Candidate, er
 func (bc *basicClient) Delete(candidate *res.Candidate) error {
 	return nil
 }
+
+// SubmitTask to jobservice
+func (bc *basicClient) SubmitTask(*res.Repository, *policy.LiteMeta) (string, error) {
+	return "", nil
+}

src/pkg/retention/launcher.go

@@ -14,11 +14,187 @@
 
 package retention
 
-import "github.com/goharbor/harbor/src/pkg/retention/policy"
+import (
+	"fmt"
+
+	"github.com/goharbor/harbor/src/common/utils"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/pkg/project"
+	"github.com/goharbor/harbor/src/pkg/repository"
+	"github.com/goharbor/harbor/src/pkg/retention/policy"
+	"github.com/goharbor/harbor/src/pkg/retention/res"
+	"github.com/goharbor/harbor/src/pkg/retention/res/selectors"
+	"github.com/pkg/errors"
+)
+
+// TODO init the client
+var client Client
 
 // Launcher provides function to launch the async jobs to run retentions based on the provided policy.
 type Launcher interface {
 	// Launch async jobs for the retention policy
 	// A separate job will be launched for each repository
-	Launch(policy *policy.Metadata) (string, []string, error)
+	//
+	//  Arguments:
+	//   policy *policy.Metadata: the policy info
+	//
+	//  Returns:
+	//   []*TaskSubmitResult : the submit results of tasks
+	//   error               : common error if any errors occurred
+	Launch(policy *policy.Metadata) ([]*TaskSubmitResult, error)
+}
+
+// NewLauncher returns an instance of Launcher
+func NewLauncher() Launcher {
+	return &launcher{}
+}
+
+type launcher struct {
+}
+
+func (l *launcher) Launch(ply *policy.Metadata) ([]*TaskSubmitResult, error) {
+	if ply == nil {
+		return nil, launcherError(fmt.Errorf("the policy is nil"))
+	}
+	// no rules, return directly
+	if len(ply.Rules) == 0 {
+		return nil, nil
+	}
+	scope := ply.Scope
+	if scope == nil {
+		return nil, launcherError(fmt.Errorf("the scope of policy is nil"))
+	}
+
+	repositoryRules := make(map[res.Repository]*policy.LiteMeta, 0)
+	level := scope.Level
+	var projectCandidates []*res.Candidate
+	var err error
+	if level == "system" {
+		// get projects
+		projectCandidates, err = getProjects()
+		if err != nil {
+			return nil, launcherError(err)
+		}
+	}
+
+	for _, rule := range ply.Rules {
+		switch level {
+		case "system":
+			// filter projects according to the project selectors
+			for _, projectSelector := range rule.ScopeSelectors["project"] {
+				selector, err := selectors.Get(projectSelector.Kind, projectSelector.Decoration,
+					projectSelector.Pattern)
+				if err != nil {
+					return nil, launcherError(err)
+				}
+				projectCandidates, err = selector.Select(projectCandidates)
+				if err != nil {
+					return nil, launcherError(err)
+				}
+			}
+		case "project":
+			projectCandidates = append(projectCandidates, &res.Candidate{
+				NamespaceID: scope.Reference,
+			})
+		}
+
+		var repositoryCandidates []*res.Candidate
+		// get repositories of projects
+		for _, projectCandidate := range projectCandidates {
+			repositories, err := getRepositories(projectCandidate.NamespaceID)
+			if err != nil {
+				return nil, launcherError(err)
+			}
+			repositoryCandidates = append(repositoryCandidates, repositories...)
+		}
+		// filter repositories according to the repository selectors
+		for _, repositorySelector := range rule.ScopeSelectors["repository"] {
+			selector, err := selectors.Get(repositorySelector.Kind, repositorySelector.Decoration,
+				repositorySelector.Pattern)
+			if err != nil {
+				return nil, launcherError(err)
+			}
+			repositoryCandidates, err = selector.Select(repositoryCandidates)
+			if err != nil {
+				return nil, launcherError(err)
+			}
+		}
+
+		for _, repositoryCandidate := range repositoryCandidates {
+			repository := res.Repository{
+				Namespace: repositoryCandidate.Namespace,
+				Name:      repositoryCandidate.Repository,
+				Kind:      repositoryCandidate.Kind,
+			}
+			if repositoryRules[repository] == nil {
+				repositoryRules[repository] = &policy.LiteMeta{
+					Algorithm: ply.Algorithm,
+				}
+			}
+			repositoryRules[repository].Rules = append(repositoryRules[repository].Rules, &rule)
+		}
+	}
+
+	var result []*TaskSubmitResult
+	for repository, rule := range repositoryRules {
+		jobID, err := client.SubmitTask(&repository, rule)
+		result = append(result, &TaskSubmitResult{
+			JobID: jobID,
+			Error: err,
+		})
+		if err != nil {
+			log.Error(launcherError(fmt.Errorf("failed to submit task: %v", err)))
+		}
+	}
+	return result, nil
+}
+
+func launcherError(err error) error {
+	return errors.Wrap(err, "launcher")
+}
+
+func getProjects() ([]*res.Candidate, error) {
+	projects, err := project.Mgr.List()
+	if err != nil {
+		return nil, err
+	}
+	var candidates []*res.Candidate
+	for _, project := range projects {
+		candidates = append(candidates, &res.Candidate{
+			NamespaceID: project.ProjectID,
+			Namespace:   project.Name,
+		})
+	}
+	return candidates, nil
+}
+
+func getRepositories(projectID int64) ([]*res.Candidate, error) {
+	var candidates []*res.Candidate
+	project, err := project.Mgr.Get(projectID)
+	if err != nil {
+		return nil, err
+	}
+	// get image repositories
+	imageRepositories, err := repository.Mgr.ListImageRepositories(projectID)
+	if err != nil {
+		return nil, err
+	}
+	for _, repository := range imageRepositories {
+		namespace, repo := utils.ParseRepository(repository.Name)
+		candidates = append(candidates, &res.Candidate{
+			Namespace:  namespace,
+			Repository: repo,
+			Kind:       "image",
+		})
+	}
+	// get chart repositories
+	chartRepositories, err := repository.Mgr.ListChartRepositories(projectID)
+	for _, repository := range chartRepositories {
+		candidates = append(candidates, &res.Candidate{
+			Namespace:  project.Name,
+			Repository: repository.Name,
+			Kind:       "chart",
+		})
+	}
+	return candidates, nil
 }

src/pkg/retention/launcher_test.go

@@ -0,0 +1,201 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package retention
+
+import (
+	"fmt"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+
+	"github.com/goharbor/harbor/src/chartserver"
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/pkg/project"
+	"github.com/goharbor/harbor/src/pkg/repository"
+	"github.com/goharbor/harbor/src/pkg/retention/policy"
+	"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
+	"github.com/goharbor/harbor/src/pkg/retention/res"
+	_ "github.com/goharbor/harbor/src/pkg/retention/res/selectors/regexp"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type fakeProjectManager struct {
+	projects []*models.Project
+}
+
+func (f *fakeProjectManager) List(...*models.ProjectQueryParam) ([]*models.Project, error) {
+	return f.projects, nil
+}
+func (f *fakeProjectManager) Get(idOrName interface{}) (*models.Project, error) {
+	id, ok := idOrName.(int64)
+	if ok {
+		for _, project := range f.projects {
+			if project.ProjectID == id {
+				return project, nil
+			}
+		}
+		return nil, nil
+	}
+	name, ok := idOrName.(string)
+	if ok {
+		for _, project := range f.projects {
+			if project.Name == name {
+				return project, nil
+			}
+		}
+		return nil, nil
+	}
+	return nil, fmt.Errorf("invalid parameter: %v, should be ID(int64) or name(string)", idOrName)
+}
+
+type fakeRepositoryManager struct {
+	imageRepositories []*models.RepoRecord
+	chartRepositories []*chartserver.ChartInfo
+}
+
+func (f *fakeRepositoryManager) ListImageRepositories(projectID int64) ([]*models.RepoRecord, error) {
+	return f.imageRepositories, nil
+}
+func (f *fakeRepositoryManager) ListChartRepositories(projectID int64) ([]*chartserver.ChartInfo, error) {
+	return f.chartRepositories, nil
+}
+
+type fakeClient struct {
+	id int
+}
+
+func (f *fakeClient) GetCandidates(repo *res.Repository) ([]*res.Candidate, error) {
+	return nil, nil
+}
+
+func (f *fakeClient) Delete(candidate *res.Candidate) error {
+	return nil
+}
+func (f *fakeClient) SubmitTask(repository *res.Repository, meta *policy.LiteMeta) (string, error) {
+	f.id++
+	return strconv.Itoa(f.id), nil
+}
+
+type launchTestSuite struct {
+	suite.Suite
+}
+
+func (l *launchTestSuite) SetupTest() {
+	pro := &models.Project{
+		ProjectID: 1,
+		Name:      "library",
+	}
+	project.Mgr = &fakeProjectManager{
+		projects: []*models.Project{
+			pro,
+		}}
+	repository.Mgr = &fakeRepositoryManager{
+		imageRepositories: []*models.RepoRecord{
+			{
+				Name: "library/image",
+			},
+		},
+		chartRepositories: []*chartserver.ChartInfo{
+			{
+				Name: "chart",
+			},
+		},
+	}
+	client = &fakeClient{}
+}
+
+func (l *launchTestSuite) TestGetProjects() {
+	projects, err := getProjects()
+	require.Nil(l.T(), err)
+	assert.Equal(l.T(), 1, len(projects))
+	assert.Equal(l.T(), int64(1), projects[0].NamespaceID)
+	assert.Equal(l.T(), "library", projects[0].Namespace)
+}
+
+func (l *launchTestSuite) TestGetRepositories() {
+	repositories, err := getRepositories(1)
+	require.Nil(l.T(), err)
+	assert.Equal(l.T(), 2, len(repositories))
+	assert.Equal(l.T(), "library", repositories[0].Namespace)
+	assert.Equal(l.T(), "image", repositories[0].Repository)
+	assert.Equal(l.T(), "image", repositories[0].Kind)
+	assert.Equal(l.T(), "library", repositories[1].Namespace)
+	assert.Equal(l.T(), "chart", repositories[1].Repository)
+	assert.Equal(l.T(), "chart", repositories[1].Kind)
+}
+
+func (l *launchTestSuite) TestLaunch() {
+	launcher := NewLauncher()
+	var ply *policy.Metadata
+	// nil policy
+	result, err := launcher.Launch(ply)
+	require.NotNil(l.T(), err)
+
+	// nil rules
+	ply = &policy.Metadata{}
+	result, err = launcher.Launch(ply)
+	require.Nil(l.T(), err)
+	assert.Equal(l.T(), 0, len(result))
+
+	// nil scope
+	ply = &policy.Metadata{
+		Rules: []rule.Metadata{
+			{},
+		},
+	}
+	_, err = launcher.Launch(ply)
+	require.NotNil(l.T(), err)
+
+	// system scope
+	ply = &policy.Metadata{
+		Scope: &policy.Scope{
+			Level: "system",
+		},
+		Rules: []rule.Metadata{
+			{
+				ScopeSelectors: map[string][]*rule.Selector{
+					"project": {
+						{
+							Kind:       "regularExpression",
+							Decoration: "matches",
+							Pattern:    "**",
+						},
+					},
+					"repository": {
+						{
+							Kind:       "regularExpression",
+							Decoration: "matches",
+							Pattern:    "**",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	result, err = launcher.Launch(ply)
+	require.Nil(l.T(), err)
+	assert.Equal(l.T(), 2, len(result))
+	assert.Equal(l.T(), "1", result[0].JobID)
+	assert.Nil(l.T(), result[0].Error)
+	assert.Equal(l.T(), "2", result[1].JobID)
+	assert.Nil(l.T(), result[1].Error)
+}
+
+func TestLaunchTestSuite(t *testing.T) {
+	suite.Run(t, new(launchTestSuite))
+}

src/pkg/retention/models.go

@@ -25,6 +25,14 @@ type Execution struct {
 	Status    string    `json:"status"`
 }
 
+// TaskSubmitResult is the result of task submitting
+// If the task is submitted successfully, JobID will be set
+// and the Error is nil
+type TaskSubmitResult struct {
+	JobID string
+	Error error
+}
+
 // History of retention
 type History struct {
 	ExecutionID string `json:"execution_id"`

src/pkg/retention/policy/models.go

@@ -67,8 +67,8 @@ type Scope struct {
 	Level string `json:"level"`
 
 	// The reference identity for the specified level
-	// '' for 'system', project ID for 'project' and repo ID for 'repository'
+	// 0 for 'system', project ID for 'project' and repo ID for 'repository'
-	Reference string `json:"ref"`
+	Reference int64 `json:"ref"`
 }
 
 // LiteMeta contains partial metadata of policy
@@ -78,5 +78,5 @@ type LiteMeta struct {
 	Algorithm string `json:"algorithm"`
 
 	// Rule collection
-	Rules []rule.Metadata `json:"rules"`
+	Rules []*rule.Metadata `json:"rules"`
 }

src/pkg/retention/policy/rule/models.go

@@ -36,7 +36,7 @@ type Metadata struct {
 	TagSelectors []*Selector `json:"tag_selectors"`
 
 	// Selector attached to the rule for filtering scope (e.g: repositories or namespaces)
-	ScopeSelectors []*Selector `json:"scope_selectors"`
+	ScopeSelectors map[string][]*Selector `json:"scope_selectors"`
 }
 
 // Selector to narrow down the list

src/pkg/retention/res/candidate.go

@@ -39,6 +39,8 @@ type Repository struct {
 
 // Candidate for retention processor to match
 type Candidate struct {
+	// Namespace(project) ID
+	NamespaceID int64
 	// Namespace
 	Namespace string
 	// Repository name