diff --git a/src/core/api/chart_repository.go b/src/core/api/chart_repository.go index e44f65174..8990671de 100644 --- a/src/core/api/chart_repository.go +++ b/src/core/api/chart_repository.go @@ -46,6 +46,11 @@ const ( // chartController is a singleton instance var chartController *chartserver.Controller +// GetChartController returns the chart controller +func GetChartController() *chartserver.Controller { + return chartController +} + // ChartRepositoryAPI provides related API handlers for the chart repository APIs type ChartRepositoryAPI struct { // The base controller to provide common utilities diff --git a/src/pkg/project/manager.go b/src/pkg/project/manager.go new file mode 100644 index 000000000..92406df4d --- /dev/null +++ b/src/pkg/project/manager.go @@ -0,0 +1,66 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package project + +import ( + "fmt" + + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" +) + +var ( + // Mgr is an instance of the default project Manager + Mgr = New() +) + +// Manager is used for project management +// currently, the interface only defines the methods needed for tag retention +// will expand it when doing refactor +type Manager interface { + // List projects according to the query + List(...*models.ProjectQueryParam) ([]*models.Project, error) + // Get the project specified by the ID or name + Get(interface{}) (*models.Project, error) +} + +// New returns a default implementation of Manager +func New() Manager { + return &manager{} +} + +type manager struct{} + +// List projects according to the query +func (m *manager) List(query ...*models.ProjectQueryParam) ([]*models.Project, error) { + var q *models.ProjectQueryParam + if len(query) > 0 { + q = query[0] + } + return dao.GetProjects(q) +} + +// Get the project specified by the ID +func (m *manager) Get(idOrName interface{}) (*models.Project, error) { + id, ok := idOrName.(int64) + if ok { + return dao.GetProjectByID(id) + } + name, ok := idOrName.(string) + if ok { + return dao.GetProjectByName(name) + } + return nil, fmt.Errorf("invalid parameter: %v, should be ID(int64) or name(string)", idOrName) +} diff --git a/src/pkg/repository/manager.go b/src/pkg/repository/manager.go new file mode 100644 index 000000000..b6b2deb72 --- /dev/null +++ b/src/pkg/repository/manager.go @@ -0,0 +1,61 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "github.com/goharbor/harbor/src/chartserver" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/core/api" + "github.com/goharbor/harbor/src/pkg/project" +) + +var ( + // Mgr is an instance of the default repository Manager + Mgr = New() +) + +// Manager is used for repository management +// currently, the interface only defines the methods needed for tag retention +// will expand it when doing refactor +type Manager interface { + // List image repositories under the project specified by the ID + ListImageRepositories(projectID int64) ([]*models.RepoRecord, error) + // List chart repositories under the project specified by the ID + ListChartRepositories(projectID int64) ([]*chartserver.ChartInfo, error) +} + +// New returns a default implementation of Manager +func New() Manager { + return &manager{} +} + +type manager struct{} + +// List image repositories under the project specified by the ID +func (m *manager) ListImageRepositories(projectID int64) ([]*models.RepoRecord, error) { + return dao.GetRepositories(&models.RepositoryQuery{ + ProjectIDs: []int64{projectID}, + }) +} + +// List chart repositories under the project specified by the ID +func (m *manager) ListChartRepositories(projectID int64) ([]*chartserver.ChartInfo, error) { + project, err := project.Mgr.Get(projectID) + if err != nil { + return nil, err + } + return api.GetChartController().ListCharts(project.Name) +} diff --git a/src/pkg/retention/client.go b/src/pkg/retention/client.go index c77deff52..af869ffb6 100644 --- a/src/pkg/retention/client.go +++ b/src/pkg/retention/client.go @@ -14,7 +14,10 @@ package retention -import "github.com/goharbor/harbor/src/pkg/retention/res" +import ( + "github.com/goharbor/harbor/src/pkg/retention/policy" + "github.com/goharbor/harbor/src/pkg/retention/res" +) // Client is designed to access core service to get required infos type Client interface { @@ -36,6 +39,17 @@ type Client interface { // Returns: // error : common error if any errors occurred Delete(candidate *res.Candidate) error + + // SubmitTask to jobservice + // + // Arguments: + // repository: *res.Repository : repository info + // meta *policy.LiteMeta : policy lite metadata + // + // Returns: + // string : the job ID + // error : common error if any errors occurred + SubmitTask(repository *res.Repository, meta *policy.LiteMeta) (string, error) } // New basic client @@ -57,3 +71,8 @@ func (bc *basicClient) GetCandidates(repo *res.Repository) ([]*res.Candidate, er func (bc *basicClient) Delete(candidate *res.Candidate) error { return nil } + +// SubmitTask to jobservice +func (bc *basicClient) SubmitTask(*res.Repository, *policy.LiteMeta) (string, error) { + return "", nil +} diff --git a/src/pkg/retention/launcher.go b/src/pkg/retention/launcher.go index 7aa1fb1ee..2d9cbbe11 100644 --- a/src/pkg/retention/launcher.go +++ b/src/pkg/retention/launcher.go @@ -14,11 +14,187 @@ package retention -import "github.com/goharbor/harbor/src/pkg/retention/policy" +import ( + "fmt" + + "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/project" + "github.com/goharbor/harbor/src/pkg/repository" + "github.com/goharbor/harbor/src/pkg/retention/policy" + "github.com/goharbor/harbor/src/pkg/retention/res" + "github.com/goharbor/harbor/src/pkg/retention/res/selectors" + "github.com/pkg/errors" +) + +// TODO init the client +var client Client // Launcher provides function to launch the async jobs to run retentions based on the provided policy. type Launcher interface { // Launch async jobs for the retention policy // A separate job will be launched for each repository - Launch(policy *policy.Metadata) (string, []string, error) + // + // Arguments: + // policy *policy.Metadata: the policy info + // + // Returns: + // []*TaskSubmitResult : the submit results of tasks + // error : common error if any errors occurred + Launch(policy *policy.Metadata) ([]*TaskSubmitResult, error) +} + +// NewLauncher returns an instance of Launcher +func NewLauncher() Launcher { + return &launcher{} +} + +type launcher struct { +} + +func (l *launcher) Launch(ply *policy.Metadata) ([]*TaskSubmitResult, error) { + if ply == nil { + return nil, launcherError(fmt.Errorf("the policy is nil")) + } + // no rules, return directly + if len(ply.Rules) == 0 { + return nil, nil + } + scope := ply.Scope + if scope == nil { + return nil, launcherError(fmt.Errorf("the scope of policy is nil")) + } + + repositoryRules := make(map[res.Repository]*policy.LiteMeta, 0) + level := scope.Level + var projectCandidates []*res.Candidate + var err error + if level == "system" { + // get projects + projectCandidates, err = getProjects() + if err != nil { + return nil, launcherError(err) + } + } + + for _, rule := range ply.Rules { + switch level { + case "system": + // filter projects according to the project selectors + for _, projectSelector := range rule.ScopeSelectors["project"] { + selector, err := selectors.Get(projectSelector.Kind, projectSelector.Decoration, + projectSelector.Pattern) + if err != nil { + return nil, launcherError(err) + } + projectCandidates, err = selector.Select(projectCandidates) + if err != nil { + return nil, launcherError(err) + } + } + case "project": + projectCandidates = append(projectCandidates, &res.Candidate{ + NamespaceID: scope.Reference, + }) + } + + var repositoryCandidates []*res.Candidate + // get repositories of projects + for _, projectCandidate := range projectCandidates { + repositories, err := getRepositories(projectCandidate.NamespaceID) + if err != nil { + return nil, launcherError(err) + } + repositoryCandidates = append(repositoryCandidates, repositories...) + } + // filter repositories according to the repository selectors + for _, repositorySelector := range rule.ScopeSelectors["repository"] { + selector, err := selectors.Get(repositorySelector.Kind, repositorySelector.Decoration, + repositorySelector.Pattern) + if err != nil { + return nil, launcherError(err) + } + repositoryCandidates, err = selector.Select(repositoryCandidates) + if err != nil { + return nil, launcherError(err) + } + } + + for _, repositoryCandidate := range repositoryCandidates { + repository := res.Repository{ + Namespace: repositoryCandidate.Namespace, + Name: repositoryCandidate.Repository, + Kind: repositoryCandidate.Kind, + } + if repositoryRules[repository] == nil { + repositoryRules[repository] = &policy.LiteMeta{ + Algorithm: ply.Algorithm, + } + } + repositoryRules[repository].Rules = append(repositoryRules[repository].Rules, &rule) + } + } + + var result []*TaskSubmitResult + for repository, rule := range repositoryRules { + jobID, err := client.SubmitTask(&repository, rule) + result = append(result, &TaskSubmitResult{ + JobID: jobID, + Error: err, + }) + if err != nil { + log.Error(launcherError(fmt.Errorf("failed to submit task: %v", err))) + } + } + return result, nil +} + +func launcherError(err error) error { + return errors.Wrap(err, "launcher") +} + +func getProjects() ([]*res.Candidate, error) { + projects, err := project.Mgr.List() + if err != nil { + return nil, err + } + var candidates []*res.Candidate + for _, project := range projects { + candidates = append(candidates, &res.Candidate{ + NamespaceID: project.ProjectID, + Namespace: project.Name, + }) + } + return candidates, nil +} + +func getRepositories(projectID int64) ([]*res.Candidate, error) { + var candidates []*res.Candidate + project, err := project.Mgr.Get(projectID) + if err != nil { + return nil, err + } + // get image repositories + imageRepositories, err := repository.Mgr.ListImageRepositories(projectID) + if err != nil { + return nil, err + } + for _, repository := range imageRepositories { + namespace, repo := utils.ParseRepository(repository.Name) + candidates = append(candidates, &res.Candidate{ + Namespace: namespace, + Repository: repo, + Kind: "image", + }) + } + // get chart repositories + chartRepositories, err := repository.Mgr.ListChartRepositories(projectID) + for _, repository := range chartRepositories { + candidates = append(candidates, &res.Candidate{ + Namespace: project.Name, + Repository: repository.Name, + Kind: "chart", + }) + } + return candidates, nil } diff --git a/src/pkg/retention/launcher_test.go b/src/pkg/retention/launcher_test.go new file mode 100644 index 000000000..394a3628c --- /dev/null +++ b/src/pkg/retention/launcher_test.go @@ -0,0 +1,201 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package retention + +import ( + "fmt" + "strconv" + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/goharbor/harbor/src/chartserver" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/pkg/project" + "github.com/goharbor/harbor/src/pkg/repository" + "github.com/goharbor/harbor/src/pkg/retention/policy" + "github.com/goharbor/harbor/src/pkg/retention/policy/rule" + "github.com/goharbor/harbor/src/pkg/retention/res" + _ "github.com/goharbor/harbor/src/pkg/retention/res/selectors/regexp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeProjectManager struct { + projects []*models.Project +} + +func (f *fakeProjectManager) List(...*models.ProjectQueryParam) ([]*models.Project, error) { + return f.projects, nil +} +func (f *fakeProjectManager) Get(idOrName interface{}) (*models.Project, error) { + id, ok := idOrName.(int64) + if ok { + for _, project := range f.projects { + if project.ProjectID == id { + return project, nil + } + } + return nil, nil + } + name, ok := idOrName.(string) + if ok { + for _, project := range f.projects { + if project.Name == name { + return project, nil + } + } + return nil, nil + } + return nil, fmt.Errorf("invalid parameter: %v, should be ID(int64) or name(string)", idOrName) +} + +type fakeRepositoryManager struct { + imageRepositories []*models.RepoRecord + chartRepositories []*chartserver.ChartInfo +} + +func (f *fakeRepositoryManager) ListImageRepositories(projectID int64) ([]*models.RepoRecord, error) { + return f.imageRepositories, nil +} +func (f *fakeRepositoryManager) ListChartRepositories(projectID int64) ([]*chartserver.ChartInfo, error) { + return f.chartRepositories, nil +} + +type fakeClient struct { + id int +} + +func (f *fakeClient) GetCandidates(repo *res.Repository) ([]*res.Candidate, error) { + return nil, nil +} + +func (f *fakeClient) Delete(candidate *res.Candidate) error { + return nil +} +func (f *fakeClient) SubmitTask(repository *res.Repository, meta *policy.LiteMeta) (string, error) { + f.id++ + return strconv.Itoa(f.id), nil +} + +type launchTestSuite struct { + suite.Suite +} + +func (l *launchTestSuite) SetupTest() { + pro := &models.Project{ + ProjectID: 1, + Name: "library", + } + project.Mgr = &fakeProjectManager{ + projects: []*models.Project{ + pro, + }} + repository.Mgr = &fakeRepositoryManager{ + imageRepositories: []*models.RepoRecord{ + { + Name: "library/image", + }, + }, + chartRepositories: []*chartserver.ChartInfo{ + { + Name: "chart", + }, + }, + } + client = &fakeClient{} +} + +func (l *launchTestSuite) TestGetProjects() { + projects, err := getProjects() + require.Nil(l.T(), err) + assert.Equal(l.T(), 1, len(projects)) + assert.Equal(l.T(), int64(1), projects[0].NamespaceID) + assert.Equal(l.T(), "library", projects[0].Namespace) +} + +func (l *launchTestSuite) TestGetRepositories() { + repositories, err := getRepositories(1) + require.Nil(l.T(), err) + assert.Equal(l.T(), 2, len(repositories)) + assert.Equal(l.T(), "library", repositories[0].Namespace) + assert.Equal(l.T(), "image", repositories[0].Repository) + assert.Equal(l.T(), "image", repositories[0].Kind) + assert.Equal(l.T(), "library", repositories[1].Namespace) + assert.Equal(l.T(), "chart", repositories[1].Repository) + assert.Equal(l.T(), "chart", repositories[1].Kind) +} + +func (l *launchTestSuite) TestLaunch() { + launcher := NewLauncher() + var ply *policy.Metadata + // nil policy + result, err := launcher.Launch(ply) + require.NotNil(l.T(), err) + + // nil rules + ply = &policy.Metadata{} + result, err = launcher.Launch(ply) + require.Nil(l.T(), err) + assert.Equal(l.T(), 0, len(result)) + + // nil scope + ply = &policy.Metadata{ + Rules: []rule.Metadata{ + {}, + }, + } + _, err = launcher.Launch(ply) + require.NotNil(l.T(), err) + + // system scope + ply = &policy.Metadata{ + Scope: &policy.Scope{ + Level: "system", + }, + Rules: []rule.Metadata{ + { + ScopeSelectors: map[string][]*rule.Selector{ + "project": { + { + Kind: "regularExpression", + Decoration: "matches", + Pattern: "**", + }, + }, + "repository": { + { + Kind: "regularExpression", + Decoration: "matches", + Pattern: "**", + }, + }, + }, + }, + }, + } + + result, err = launcher.Launch(ply) + require.Nil(l.T(), err) + assert.Equal(l.T(), 2, len(result)) + assert.Equal(l.T(), "1", result[0].JobID) + assert.Nil(l.T(), result[0].Error) + assert.Equal(l.T(), "2", result[1].JobID) + assert.Nil(l.T(), result[1].Error) +} + +func TestLaunchTestSuite(t *testing.T) { + suite.Run(t, new(launchTestSuite)) +} diff --git a/src/pkg/retention/models.go b/src/pkg/retention/models.go index 958685782..f2a190500 100644 --- a/src/pkg/retention/models.go +++ b/src/pkg/retention/models.go @@ -25,6 +25,14 @@ type Execution struct { Status string `json:"status"` } +// TaskSubmitResult is the result of task submitting +// If the task is submitted successfully, JobID will be set +// and the Error is nil +type TaskSubmitResult struct { + JobID string + Error error +} + // History of retention type History struct { ExecutionID string `json:"execution_id"` diff --git a/src/pkg/retention/policy/models.go b/src/pkg/retention/policy/models.go index 4aa72adc9..ae0e764f8 100644 --- a/src/pkg/retention/policy/models.go +++ b/src/pkg/retention/policy/models.go @@ -67,8 +67,8 @@ type Scope struct { Level string `json:"level"` // The reference identity for the specified level - // '' for 'system', project ID for 'project' and repo ID for 'repository' - Reference string `json:"ref"` + // 0 for 'system', project ID for 'project' and repo ID for 'repository' + Reference int64 `json:"ref"` } // LiteMeta contains partial metadata of policy @@ -78,5 +78,5 @@ type LiteMeta struct { Algorithm string `json:"algorithm"` // Rule collection - Rules []rule.Metadata `json:"rules"` + Rules []*rule.Metadata `json:"rules"` } diff --git a/src/pkg/retention/policy/rule/models.go b/src/pkg/retention/policy/rule/models.go index c853fb962..b14841e72 100644 --- a/src/pkg/retention/policy/rule/models.go +++ b/src/pkg/retention/policy/rule/models.go @@ -36,7 +36,7 @@ type Metadata struct { TagSelectors []*Selector `json:"tag_selectors"` // Selector attached to the rule for filtering scope (e.g: repositories or namespaces) - ScopeSelectors []*Selector `json:"scope_selectors"` + ScopeSelectors map[string][]*Selector `json:"scope_selectors"` } // Selector to narrow down the list diff --git a/src/pkg/retention/res/candidate.go b/src/pkg/retention/res/candidate.go index 6b57f4d00..db8004079 100644 --- a/src/pkg/retention/res/candidate.go +++ b/src/pkg/retention/res/candidate.go @@ -39,6 +39,8 @@ type Repository struct { // Candidate for retention processor to match type Candidate struct { + // Namespace(project) ID + NamespaceID int64 // Namespace Namespace string // Repository name