Merge branch 'feature/tag_retention' into feature/tag_retention-performer

src/common/models/repo.go

@@ -16,6 +16,9 @@ package models
 
 import (
 	"time"
+
+	"github.com/goharbor/harbor/src/common/utils/notary/model"
+	"github.com/theupdateframework/notary/tuf/data"
 )
 
 // RepoTable is the table name for repository
@@ -47,3 +50,36 @@ type RepositoryQuery struct {
 	Pagination
 	Sorting
 }
+
+// TagResp holds the information of one image tag
+type TagResp struct {
+	TagDetail
+	Signature    *model.Target    `json:"signature"`
+	ScanOverview *ImgScanOverview `json:"scan_overview,omitempty"`
+	Labels       []*Label         `json:"labels"`
+}
+
+// TagDetail ...
+type TagDetail struct {
+	Digest        string    `json:"digest"`
+	Name          string    `json:"name"`
+	Size          int64     `json:"size"`
+	Architecture  string    `json:"architecture"`
+	OS            string    `json:"os"`
+	OSVersion     string    `json:"os.version"`
+	DockerVersion string    `json:"docker_version"`
+	Author        string    `json:"author"`
+	Created       time.Time `json:"created"`
+	Config        *TagCfg   `json:"config"`
+}
+
+// TagCfg ...
+type TagCfg struct {
+	Labels map[string]string `json:"labels"`
+}
+
+// Signature ...
+type Signature struct {
+	Tag    string      `json:"tag"`
+	Hashes data.Hashes `json:"hashes"`
+}

src/common/utils/notary/helper.go

@@ -22,6 +22,8 @@ import (
 	"path"
 	"strings"
 
+	"github.com/goharbor/harbor/src/common/utils/notary/model"
+
 	"github.com/docker/distribution/registry/auth/token"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/common/utils/registry"
@@ -41,14 +43,6 @@ var (
 	mockRetriever   notary.PassRetriever
 )
 
-// Target represents the json object of a target of a docker image in notary.
-// The struct will be used when repository is know so it won'g contain the name of a repository.
-type Target struct {
-	Tag    string      `json:"tag"`
-	Hashes data.Hashes `json:"hashes"`
-	// TODO: update fields as needed.
-}
-
 func init() {
 	mockRetriever = func(keyName, alias string, createNew bool, attempts int) (passphrase string, giveup bool, err error) {
 		passphrase = "hardcode"
@@ -60,7 +54,7 @@ func init() {
 }
 
 // GetInternalTargets wraps GetTargets to read config values for getting full-qualified repo from internal notary instance.
-func GetInternalTargets(notaryEndpoint string, username string, repo string) ([]Target, error) {
+func GetInternalTargets(notaryEndpoint string, username string, repo string) ([]model.Target, error) {
 	ext, err := config.ExtEndpoint()
 	if err != nil {
 		log.Errorf("Error while reading external endpoint: %v", err)
@@ -74,8 +68,8 @@ func GetInternalTargets(notaryEndpoint string, username string, repo string) ([]
 // GetTargets is a help function called by API to fetch signature information of a given repository.
 // Per docker's convention the repository should contain the information of endpoint, i.e. it should look
 // like "192.168.0.1/library/ubuntu", instead of "library/ubuntu" (fqRepo for fully-qualified repo)
-func GetTargets(notaryEndpoint string, username string, fqRepo string) ([]Target, error) {
+func GetTargets(notaryEndpoint string, username string, fqRepo string) ([]model.Target, error) {
-	res := []Target{}
+	res := []model.Target{}
 	t, err := tokenutil.MakeToken(username, tokenutil.Notary,
 		[]*token.ResourceActions{
 			{
@@ -109,13 +103,16 @@ func GetTargets(notaryEndpoint string, username string, fqRepo string) ([]Target
 		log.Warningf("Failed to clear cached root.json: %s, error: %v, when repo is removed from notary the signature status maybe incorrect", rootJSON, rmErr)
 	}
 	for _, t := range targets {
-		res = append(res, Target{t.Name, t.Hashes})
+		res = append(res, model.Target{
+			Tag:    t.Name,
+			Hashes: t.Hashes,
+		})
 	}
 	return res, nil
 }
 
 // DigestFromTarget get a target and return the value of digest, in accordance to Docker-Content-Digest
-func DigestFromTarget(t Target) (string, error) {
+func DigestFromTarget(t model.Target) (string, error) {
 	sha, ok := t.Hashes["sha256"]
 	if !ok {
 		return "", fmt.Errorf("no valid hash, expecting sha256")

src/common/utils/notary/helper_test.go

@@ -17,6 +17,8 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/goharbor/harbor/src/common/utils/notary/model"
+
 	notarytest "github.com/goharbor/harbor/src/common/utils/notary/test"
 	"github.com/goharbor/harbor/src/common/utils/test"
 	"github.com/goharbor/harbor/src/core/config"
@@ -81,17 +83,19 @@ func TestGetDigestFromTarget(t *testing.T) {
 				}
 		}`
 
-	var t1 Target
+	var t1 model.Target
 	err := json.Unmarshal([]byte(str), &t1)
 	if err != nil {
 		panic(err)
 	}
 	hash2 := make(map[string][]byte)
-	t2 := Target{"2.0", hash2}
+	t2 := model.Target{
+		Tag:    "2.0",
+		Hashes: hash2,
+	}
 	d1, err1 := DigestFromTarget(t1)
 	assert.Nil(t, err1, "Unexpected error: %v", err1)
 	assert.Equal(t, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7", d1, "digest mismatch")
 	_, err2 := DigestFromTarget(t2)
 	assert.NotNil(t, err2, "")
-
 }

src/common/utils/notary/model/model.go

@@ -0,0 +1,25 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package model
+
+import "github.com/theupdateframework/notary/tuf/data"
+
+// Target represents the json object of a target of a docker image in notary.
+// The struct will be used when repository is know so it won'g contain the name of a repository.
+type Target struct {
+	Tag    string      `json:"tag"`
+	Hashes data.Hashes `json:"hashes"`
+	// TODO: update fields as needed.
+}

src/core/api/harborapi_test.go

@@ -555,7 +555,7 @@ func (a testapi) GetRepos(authInfo usrInfo, projectID, keyword string) (
 	return code, nil, nil
 }
 
-func (a testapi) GetTag(authInfo usrInfo, repository string, tag string) (int, *TagResp, error) {
+func (a testapi) GetTag(authInfo usrInfo, repository string, tag string) (int, *models.TagResp, error) {
 	_sling := sling.New().Get(a.basePath).Path(fmt.Sprintf("/api/repositories/%s/tags/%s", repository, tag))
 	code, data, err := request(_sling, jsonAcceptHeader, authInfo)
 	if err != nil {
@@ -567,7 +567,7 @@ func (a testapi) GetTag(authInfo usrInfo, repository string, tag string) (int, *
 		return code, nil, nil
 	}
 
-	result := TagResp{}
+	result := models.TagResp{}
 	if err := json.Unmarshal(data, &result); err != nil {
 		return 0, nil, err
 	}
@@ -591,7 +591,7 @@ func (a testapi) GetReposTags(authInfo usrInfo, repoName string) (int, interface
 		return httpStatusCode, body, nil
 	}
 
-	result := []TagResp{}
+	result := []models.TagResp{}
 	if err := json.Unmarshal(body, &result); err != nil {
 		return 0, nil, err
 	}

src/core/api/repository.go

@@ -16,6 +16,7 @@ package api
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -24,10 +25,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/goharbor/harbor/src/pkg/scan"
-
-	"errors"
-
 	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/goharbor/harbor/src/common"
@@ -38,9 +35,11 @@ import (
 	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/common/utils/notary"
+	notarymodel "github.com/goharbor/harbor/src/common/utils/notary/model"
 	"github.com/goharbor/harbor/src/common/utils/registry"
 	"github.com/goharbor/harbor/src/core/config"
 	coreutils "github.com/goharbor/harbor/src/core/utils"
+	"github.com/goharbor/harbor/src/pkg/scan"
 	"github.com/goharbor/harbor/src/replication"
 	"github.com/goharbor/harbor/src/replication/event"
 	"github.com/goharbor/harbor/src/replication/model"
@@ -80,31 +79,6 @@ func (r reposSorter) Less(i, j int) bool {
 	return r[i].Index < r[j].Index
 }
 
-type tagDetail struct {
-	Digest        string    `json:"digest"`
-	Name          string    `json:"name"`
-	Size          int64     `json:"size"`
-	Architecture  string    `json:"architecture"`
-	OS            string    `json:"os"`
-	OSVersion     string    `json:"os.version"`
-	DockerVersion string    `json:"docker_version"`
-	Author        string    `json:"author"`
-	Created       time.Time `json:"created"`
-	Config        *cfg      `json:"config"`
-}
-
-type cfg struct {
-	Labels map[string]string `json:"labels"`
-}
-
-// TagResp holds the information of one image tag
-type TagResp struct {
-	tagDetail
-	Signature    *notary.Target          `json:"signature"`
-	ScanOverview *models.ImgScanOverview `json:"scan_overview,omitempty"`
-	Labels       []*models.Label         `json:"labels"`
-}
-
 type manifestResp struct {
 	Manifest interface{} `json:"manifest"`
 	Config   interface{} `json:"config,omitempty" `
@@ -610,24 +584,24 @@ func (ra *RepositoryAPI) GetTags() {
 // get config, signature and scan overview and assemble them into one
 // struct for each tag in tags
 func assembleTagsInParallel(client *registry.Repository, repository string,
-	tags []string, username string) []*TagResp {
+	tags []string, username string) []*models.TagResp {
 	var err error
-	signatures := map[string][]notary.Target{}
+	signatures := map[string][]notarymodel.Target{}
 	if config.WithNotary() {
 		signatures, err = getSignatures(username, repository)
 		if err != nil {
-			signatures = map[string][]notary.Target{}
+			signatures = map[string][]notarymodel.Target{}
 			log.Errorf("failed to get signatures of %s: %v", repository, err)
 		}
 	}
 
-	c := make(chan *TagResp)
+	c := make(chan *models.TagResp)
 	for _, tag := range tags {
 		go assembleTag(c, client, repository, tag, config.WithClair(),
 			config.WithNotary(), signatures)
 	}
-	result := []*TagResp{}
+	result := []*models.TagResp{}
-	var item *TagResp
+	var item *models.TagResp
 	for i := 0; i < len(tags); i++ {
 		item = <-c
 		if item == nil {
@@ -638,10 +612,10 @@ func assembleTagsInParallel(client *registry.Repository, repository string,
 	return result
 }
 
-func assembleTag(c chan *TagResp, client *registry.Repository,
+func assembleTag(c chan *models.TagResp, client *registry.Repository,
 	repository, tag string, clairEnabled, notaryEnabled bool,
-	signatures map[string][]notary.Target) {
+	signatures map[string][]notarymodel.Target) {
-	item := &TagResp{}
+	item := &models.TagResp{}
 	// labels
 	image := fmt.Sprintf("%s:%s", repository, tag)
 	labels, err := dao.GetLabelsOfResource(common.ResourceTypeImage, image)
@@ -657,7 +631,7 @@ func assembleTag(c chan *TagResp, client *registry.Repository,
 		log.Errorf("failed to get v2 manifest of %s:%s: %v", repository, tag, err)
 	}
 	if tagDetail != nil {
-		item.tagDetail = *tagDetail
+		item.TagDetail = *tagDetail
 	}
 
 	// scan overview
@@ -680,8 +654,8 @@ func assembleTag(c chan *TagResp, client *registry.Repository,
 
 // getTagDetail returns the detail information for v2 manifest image
 // The information contains architecture, os, author, size, etc.
-func getTagDetail(client *registry.Repository, tag string) (*tagDetail, error) {
+func getTagDetail(client *registry.Repository, tag string) (*models.TagDetail, error) {
-	detail := &tagDetail{
+	detail := &models.TagDetail{
 		Name: tag,
 	}
 
@@ -738,7 +712,7 @@ func getTagDetail(client *registry.Repository, tag string) (*tagDetail, error) {
 	return detail, nil
 }
 
-func populateAuthor(detail *tagDetail) {
+func populateAuthor(detail *models.TagDetail) {
 	// has author info already
 	if len(detail.Author) > 0 {
 		return
@@ -1046,14 +1020,14 @@ func (ra *RepositoryAPI) VulnerabilityDetails() {
 	ra.ServeJSON()
 }
 
-func getSignatures(username, repository string) (map[string][]notary.Target, error) {
+func getSignatures(username, repository string) (map[string][]notarymodel.Target, error) {
 	targets, err := notary.GetInternalTargets(config.InternalNotaryEndpoint(),
 		username, repository)
 	if err != nil {
 		return nil, err
 	}
 
-	signatures := map[string][]notary.Target{}
+	signatures := map[string][]notarymodel.Target{}
 	for _, tgt := range targets {
 		digest, err := notary.DigestFromTarget(tgt)
 		if err != nil {

src/core/api/repository_test.go

@@ -96,7 +96,7 @@ func TestGetReposTags(t *testing.T) {
 		t.Errorf("failed to get tags of repository %s: %v", repository, err)
 	} else {
 		assert.Equal(int(200), code, "httpStatusCode should be 200")
-		if tg, ok := tags.([]TagResp); ok {
+		if tg, ok := tags.([]models.TagResp); ok {
 			assert.Equal(1, len(tg), fmt.Sprintf("there should be only one tag, but now %v", tg))
 			assert.Equal(tg[0].Name, "latest", "the tag should be latest")
 		} else {
@@ -207,19 +207,19 @@ func TestGetReposTop(t *testing.T) {
 
 func TestPopulateAuthor(t *testing.T) {
 	author := "author"
-	detail := &tagDetail{
+	detail := &models.TagDetail{
 		Author: author,
 	}
 	populateAuthor(detail)
 	assert.Equal(t, author, detail.Author)
 
-	detail = &tagDetail{}
+	detail = &models.TagDetail{}
 	populateAuthor(detail)
 	assert.Equal(t, "", detail.Author)
 
 	maintainer := "maintainer"
-	detail = &tagDetail{
+	detail = &models.TagDetail{
-		Config: &cfg{
+		Config: &models.TagCfg{
 			Labels: map[string]string{
 				"Maintainer": maintainer,
 			},

src/pkg/clients/core/client.go

@@ -18,10 +18,11 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/goharbor/harbor/src/common/models"
+
 	"github.com/goharbor/harbor/src/chartserver"
 	chttp "github.com/goharbor/harbor/src/common/http"
 	"github.com/goharbor/harbor/src/common/http/modifier"
-	"github.com/goharbor/harbor/src/core/api"
 )
 
 // Client defines the methods that a core client should implement
@@ -34,7 +35,7 @@ type Client interface {
 
 // ImageClient defines the methods that an image client should implement
 type ImageClient interface {
-	ListAllImages(project, repository string) ([]*api.TagResp, error)
+	ListAllImages(project, repository string) ([]*models.TagResp, error)
 	DeleteImage(project, repository, tag string) error
 }
 

src/pkg/clients/core/image.go

@@ -17,12 +17,12 @@ package core
 import (
 	"fmt"
 
-	"github.com/goharbor/harbor/src/core/api"
+	"github.com/goharbor/harbor/src/common/models"
 )
 
-func (c *client) ListAllImages(project, repository string) ([]*api.TagResp, error) {
+func (c *client) ListAllImages(project, repository string) ([]*models.TagResp, error) {
 	url := c.buildURL(fmt.Sprintf("/api/repositories/%s/%s/tags", project, repository))
-	var images []*api.TagResp
+	var images []*models.TagResp
 	if err := c.httpclient.GetAndIteratePagination(url, &images); err != nil {
 		return nil, err
 	}

src/pkg/retention/client_test.go

@@ -18,8 +18,8 @@ import (
 	"testing"
 
 	"github.com/goharbor/harbor/src/chartserver"
-	"github.com/goharbor/harbor/src/common/job/models"
+	jmodels "github.com/goharbor/harbor/src/common/job/models"
-	"github.com/goharbor/harbor/src/core/api"
+	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/pkg/retention/res"
 	"github.com/goharbor/harbor/src/testing/clients"
@@ -34,10 +34,10 @@ type fakeCoreClient struct {
 	clients.DumbCoreClient
 }
 
-func (f *fakeCoreClient) ListAllImages(project, repository string) ([]*api.TagResp, error) {
+func (f *fakeCoreClient) ListAllImages(project, repository string) ([]*models.TagResp, error) {
-	image := &api.TagResp{}
+	image := &models.TagResp{}
 	image.Name = "latest"
-	return []*api.TagResp{image}, nil
+	return []*models.TagResp{image}, nil
 }
 
 func (f *fakeCoreClient) ListAllCharts(project, repository string) ([]*chartserver.ChartVersion, error) {
@@ -53,7 +53,7 @@ func (f *fakeCoreClient) ListAllCharts(project, repository string) ([]*chartserv
 
 type fakeJobserviceClient struct{}
 
-func (f *fakeJobserviceClient) SubmitJob(*models.JobData) (string, error) {
+func (f *fakeJobserviceClient) SubmitJob(*jmodels.JobData) (string, error) {
 	return "1", nil
 }
 func (f *fakeJobserviceClient) GetJobLog(uuid string) ([]byte, error) {

src/pkg/retention/launcher_test.go

@@ -216,15 +216,15 @@ func (l *launchTestSuite) TestLaunch() {
 				ScopeSelectors: map[string][]*rule.Selector{
 					"project": {
 						{
-							Kind:       "regularExpression",
+							Kind:       "doublestar",
-							Decoration: "matches",
+							Decoration: "nsMatches",
 							Pattern:    "**",
 						},
 					},
 					"repository": {
 						{
-							Kind:       "regularExpression",
+							Kind:       "doublestar",
-							Decoration: "matches",
+							Decoration: "repoMatches",
 							Pattern:    "**",
 						},
 					},

src/pkg/retention/policy/alg/or/processor_test.go

@@ -102,7 +102,7 @@ func (suite *ProcessorTestSuite) TearDownSuite() {}
 func (suite *ProcessorTestSuite) TestProcess() {
 	results, err := suite.p.Process(suite.all)
 	require.NoError(suite.T(), err)
-	assert.Equal(suite.T(), 2, len(results))
+	assert.Equal(suite.T(), 1, len(results))
 	assert.Condition(suite.T(), func() bool {
 		for _, r := range results {
 			if r.Error != nil {

src/pkg/retention/res/selectors/doublestar/selector.go

@@ -31,6 +31,10 @@ const (
 	RepoMatches = "repoMatches"
 	// RepoExcludes represents repository excludes [pattern]
 	RepoExcludes = "repoExcludes"
+	// NSMatches represents namespace matches [pattern]
+	NSMatches = "nsMatches"
+	// NSExcludes represents namespace excludes [pattern]
+	NSExcludes = "nsExcludes"
 )
 
 // selector for regular expression
@@ -59,6 +63,11 @@ func (s *selector) Select(artifacts []*res.Candidate) (selected []*res.Candidate
 		case RepoExcludes:
 			value = art.Repository
 			excludes = true
+		case NSMatches:
+			value = art.Namespace
+		case NSExcludes:
+			value = art.Namespace
+			excludes = true
 		}
 
 		if len(value) > 0 {
@@ -95,5 +104,12 @@ func match(pattern, str string) (bool, error) {
 
 func init() {
 	// Register doublestar selector
-	selectors.Register(Kind, []string{Matches, Excludes}, New)
+	selectors.Register(Kind, []string{
+		Matches,
+		Excludes,
+		RepoMatches,
+		RepoExcludes,
+		NSMatches,
+		NSExcludes,
+	}, New)
 }

src/pkg/retention/res/selectors/doublestar/selector_test.go

@@ -51,8 +51,8 @@ func (suite *RegExpSelectorTestSuite) SetupSuite() {
 			Labels:       []string{"label1", "label2", "label3"},
 		},
 		{
-			NamespaceID:  1,
+			NamespaceID:  2,
-			Namespace:    "library",
+			Namespace:    "retention",
 			Repository:   "redis",
 			Tag:          "4.0",
 			Kind:         res.Image,
@@ -62,8 +62,8 @@ func (suite *RegExpSelectorTestSuite) SetupSuite() {
 			Labels:       []string{"label1", "label4", "label5"},
 		},
 		{
-			NamespaceID:  1,
+			NamespaceID:  2,
-			Namespace:    "library",
+			Namespace:    "retention",
 			Repository:   "redis",
 			Tag:          "4.1",
 			Kind:         res.Image,
@@ -180,6 +180,60 @@ func (suite *RegExpSelectorTestSuite) TestRepoExcludes() {
 	})
 }
 
+// TestNSMatches tests the namespace `matches` case
+func (suite *RegExpSelectorTestSuite) TestNSMatches() {
+	repoMatches := &selector{
+		decoration: NSMatches,
+		pattern:    "{library}",
+	}
+
+	selected, err := repoMatches.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 1, len(selected))
+	assert.Condition(suite.T(), func() bool {
+		return expect([]string{"harbor:latest"}, selected)
+	})
+
+	repoMatches2 := &selector{
+		decoration: RepoMatches,
+		pattern:    "re*",
+	}
+
+	selected, err = repoMatches2.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 2, len(selected))
+	assert.Condition(suite.T(), func() bool {
+		return expect([]string{"redis:4.0", "redis:4.1"}, selected)
+	})
+}
+
+// TestNSExcludes tests the namespace `excludes` case
+func (suite *RegExpSelectorTestSuite) TestNSExcludes() {
+	repoExcludes := &selector{
+		decoration: NSExcludes,
+		pattern:    "{library}",
+	}
+
+	selected, err := repoExcludes.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 2, len(selected))
+	assert.Condition(suite.T(), func() bool {
+		return expect([]string{"redis:4.0", "redis:4.1"}, selected)
+	})
+
+	repoExcludes2 := &selector{
+		decoration: NSExcludes,
+		pattern:    "re*",
+	}
+
+	selected, err = repoExcludes2.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 1, len(selected))
+	assert.Condition(suite.T(), func() bool {
+		return expect([]string{"harbor:latest"}, selected)
+	})
+}
+
 // Check whether the returned result matched the expected ones (only check repo:tag)
 func expect(expected []string, candidates []*res.Candidate) bool {
 	hash := make(map[string]bool)

src/testing/clients/dumb_core_client.go

@@ -16,7 +16,7 @@ package clients
 
 import (
 	"github.com/goharbor/harbor/src/chartserver"
-	"github.com/goharbor/harbor/src/core/api"
+	"github.com/goharbor/harbor/src/common/models"
 )
 
 // DumbCoreClient provides an empty implement for pkg/clients/core.Client
@@ -24,7 +24,7 @@ import (
 type DumbCoreClient struct{}
 
 // ListAllImages ...
-func (d *DumbCoreClient) ListAllImages(project, repository string) ([]*api.TagResp, error) {
+func (d *DumbCoreClient) ListAllImages(project, repository string) ([]*models.TagResp, error) {
 	return nil, nil
 }