Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-24 09:38:09 +01:00
Code Issues Projects Releases Wiki Activity

Merge branch 'feature/tag_retention' into feature/tag_retention-performer

Browse Source
This commit is contained in:
Steven Zou 2019-07-19 16:00:13 +08:00
commit f0ea62caa9
15 changed files with 198 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/common/models/repo.go
View File

@ -16,6 +16,9 @@ package models
import (
"time"
"github.com/goharbor/harbor/src/common/utils/notary/model"
"github.com/theupdateframework/notary/tuf/data"
)
// RepoTable is the table name for repository
@ -47,3 +50,36 @@ type RepositoryQuery struct {
Pagination
Sorting
}
// TagResp holds the information of one image tag
type TagResp struct {
TagDetail
Signature *model.Target `json:"signature"`
ScanOverview *ImgScanOverview `json:"scan_overview,omitempty"`
Labels []*Label `json:"labels"`
}
// TagDetail ...
type TagDetail struct {
Digest string `json:"digest"`
Name string `json:"name"`
Size int64 `json:"size"`
Architecture string `json:"architecture"`
OS string `json:"os"`
OSVersion string `json:"os.version"`
DockerVersion string `json:"docker_version"`
Author string `json:"author"`
Created time.Time `json:"created"`
Config *TagCfg `json:"config"`
}
// TagCfg ...
type TagCfg struct {
Labels map[string]string `json:"labels"`
}
// Signature ...
type Signature struct {
Tag string `json:"tag"`
Hashes data.Hashes `json:"hashes"`
}

23
src/common/utils/notary/helper.go
View File

@ -22,6 +22,8 @@ import (
"path"
"strings"
"github.com/goharbor/harbor/src/common/utils/notary/model"
"github.com/docker/distribution/registry/auth/token"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry"
@ -41,14 +43,6 @@ var (
mockRetriever notary.PassRetriever
)
// Target represents the json object of a target of a docker image in notary.
// The struct will be used when repository is know so it won'g contain the name of a repository.
type Target struct {
Tag string `json:"tag"`
Hashes data.Hashes `json:"hashes"`
// TODO: update fields as needed.
}
func init() {
mockRetriever = func(keyName, alias string, createNew bool, attempts int) (passphrase string, giveup bool, err error) {
passphrase = "hardcode"
@ -60,7 +54,7 @@ func init() {
}
// GetInternalTargets wraps GetTargets to read config values for getting full-qualified repo from internal notary instance.
func GetInternalTargets(notaryEndpoint string, username string, repo string) ([]Target, error) {
func GetInternalTargets(notaryEndpoint string, username string, repo string) ([]model.Target, error) {
ext, err := config.ExtEndpoint()
if err != nil {
log.Errorf("Error while reading external endpoint: %v", err)
@ -74,8 +68,8 @@ func GetInternalTargets(notaryEndpoint string, username string, repo string) ([]
// GetTargets is a help function called by API to fetch signature information of a given repository.
// Per docker's convention the repository should contain the information of endpoint, i.e. it should look
// like "192.168.0.1/library/ubuntu", instead of "library/ubuntu" (fqRepo for fully-qualified repo)
func GetTargets(notaryEndpoint string, username string, fqRepo string) ([]Target, error) {
res := []Target{}
func GetTargets(notaryEndpoint string, username string, fqRepo string) ([]model.Target, error) {
res := []model.Target{}
t, err := tokenutil.MakeToken(username, tokenutil.Notary,
[]*token.ResourceActions{
{
@ -109,13 +103,16 @@ func GetTargets(notaryEndpoint string, username string, fqRepo string) ([]Target
log.Warningf("Failed to clear cached root.json: %s, error: %v, when repo is removed from notary the signature status maybe incorrect", rootJSON, rmErr)
}
for _, t := range targets {
res = append(res, Target{t.Name, t.Hashes})
res = append(res, model.Target{
Tag: t.Name,
Hashes: t.Hashes,
})
}
return res, nil
}
// DigestFromTarget get a target and return the value of digest, in accordance to Docker-Content-Digest
func DigestFromTarget(t Target) (string, error) {
func DigestFromTarget(t model.Target) (string, error) {
sha, ok := t.Hashes["sha256"]
if !ok {
return "", fmt.Errorf("no valid hash, expecting sha256")

10
src/common/utils/notary/helper_test.go
View File

@ -17,6 +17,8 @@ import (
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/common/utils/notary/model"
notarytest "github.com/goharbor/harbor/src/common/utils/notary/test"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/core/config"
@ -81,17 +83,19 @@ func TestGetDigestFromTarget(t *testing.T) {
}
}`
var t1 Target
var t1 model.Target
err := json.Unmarshal([]byte(str), &t1)
if err != nil {
panic(err)
}
hash2 := make(map[string][]byte)
t2 := Target{"2.0", hash2}
t2 := model.Target{
Tag: "2.0",
Hashes: hash2,
}
d1, err1 := DigestFromTarget(t1)
assert.Nil(t, err1, "Unexpected error: %v", err1)
assert.Equal(t, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7", d1, "digest mismatch")
_, err2 := DigestFromTarget(t2)
assert.NotNil(t, err2, "")
}

25
src/common/utils/notary/model/model.go Normal file
View File

@ -0,0 +1,25 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
import "github.com/theupdateframework/notary/tuf/data"
// Target represents the json object of a target of a docker image in notary.
// The struct will be used when repository is know so it won'g contain the name of a repository.
type Target struct {
Tag string `json:"tag"`
Hashes data.Hashes `json:"hashes"`
// TODO: update fields as needed.
}

6
src/core/api/harborapi_test.go
View File

@ -555,7 +555,7 @@ func (a testapi) GetRepos(authInfo usrInfo, projectID, keyword string) (
return code, nil, nil
}
func (a testapi) GetTag(authInfo usrInfo, repository string, tag string) (int, *TagResp, error) {
func (a testapi) GetTag(authInfo usrInfo, repository string, tag string) (int, *models.TagResp, error) {
_sling := sling.New().Get(a.basePath).Path(fmt.Sprintf("/api/repositories/%s/tags/%s", repository, tag))
code, data, err := request(_sling, jsonAcceptHeader, authInfo)
if err != nil {
@ -567,7 +567,7 @@ func (a testapi) GetTag(authInfo usrInfo, repository string, tag string) (int, *
return code, nil, nil
}
result := TagResp{}
result := models.TagResp{}
if err := json.Unmarshal(data, &result); err != nil {
return 0, nil, err
}
@ -591,7 +591,7 @@ func (a testapi) GetReposTags(authInfo usrInfo, repoName string) (int, interface
return httpStatusCode, body, nil
}
result := []TagResp{}
result := []models.TagResp{}
if err := json.Unmarshal(body, &result); err != nil {
return 0, nil, err
}

62
src/core/api/repository.go
View File

@ -16,6 +16,7 @@ package api
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
@ -24,10 +25,6 @@ import (
"strings"
"time"
"github.com/goharbor/harbor/src/pkg/scan"
"errors"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common"
@ -38,9 +35,11 @@ import (
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/notary"
notarymodel "github.com/goharbor/harbor/src/common/utils/notary/model"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/core/config"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/pkg/scan"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/model"
@ -80,31 +79,6 @@ func (r reposSorter) Less(i, j int) bool {
return r[i].Index < r[j].Index
}
type tagDetail struct {
Digest string `json:"digest"`
Name string `json:"name"`
Size int64 `json:"size"`
Architecture string `json:"architecture"`
OS string `json:"os"`
OSVersion string `json:"os.version"`
DockerVersion string `json:"docker_version"`
Author string `json:"author"`
Created time.Time `json:"created"`
Config *cfg `json:"config"`
}
type cfg struct {
Labels map[string]string `json:"labels"`
}
// TagResp holds the information of one image tag
type TagResp struct {
tagDetail
Signature *notary.Target `json:"signature"`
ScanOverview *models.ImgScanOverview `json:"scan_overview,omitempty"`
Labels []*models.Label `json:"labels"`
}
type manifestResp struct {
Manifest interface{} `json:"manifest"`
Config interface{} `json:"config,omitempty" `
@ -610,24 +584,24 @@ func (ra *RepositoryAPI) GetTags() {
// get config, signature and scan overview and assemble them into one
// struct for each tag in tags
func assembleTagsInParallel(client *registry.Repository, repository string,
tags []string, username string) []*TagResp {
tags []string, username string) []*models.TagResp {
var err error
signatures := map[string][]notary.Target{}
signatures := map[string][]notarymodel.Target{}
if config.WithNotary() {
signatures, err = getSignatures(username, repository)
if err != nil {
signatures = map[string][]notary.Target{}
signatures = map[string][]notarymodel.Target{}
log.Errorf("failed to get signatures of %s: %v", repository, err)
}
}
c := make(chan *TagResp)
c := make(chan *models.TagResp)
for _, tag := range tags {
go assembleTag(c, client, repository, tag, config.WithClair(),
config.WithNotary(), signatures)
}
result := []*TagResp{}
var item *TagResp
result := []*models.TagResp{}
var item *models.TagResp
for i := 0; i < len(tags); i++ {
item = <-c
if item == nil {
@ -638,10 +612,10 @@ func assembleTagsInParallel(client *registry.Repository, repository string,
return result
}
func assembleTag(c chan *TagResp, client *registry.Repository,
func assembleTag(c chan *models.TagResp, client *registry.Repository,
repository, tag string, clairEnabled, notaryEnabled bool,
signatures map[string][]notary.Target) {
item := &TagResp{}
signatures map[string][]notarymodel.Target) {
item := &models.TagResp{}
// labels
image := fmt.Sprintf("%s:%s", repository, tag)
labels, err := dao.GetLabelsOfResource(common.ResourceTypeImage, image)
@ -657,7 +631,7 @@ func assembleTag(c chan *TagResp, client *registry.Repository,
log.Errorf("failed to get v2 manifest of %s:%s: %v", repository, tag, err)
}
if tagDetail != nil {
item.tagDetail = *tagDetail
item.TagDetail = *tagDetail
}
// scan overview
@ -680,8 +654,8 @@ func assembleTag(c chan *TagResp, client *registry.Repository,
// getTagDetail returns the detail information for v2 manifest image
// The information contains architecture, os, author, size, etc.
func getTagDetail(client *registry.Repository, tag string) (*tagDetail, error) {
detail := &tagDetail{
func getTagDetail(client *registry.Repository, tag string) (*models.TagDetail, error) {
detail := &models.TagDetail{
Name: tag,
}
@ -738,7 +712,7 @@ func getTagDetail(client *registry.Repository, tag string) (*tagDetail, error) {
return detail, nil
}
func populateAuthor(detail *tagDetail) {
func populateAuthor(detail *models.TagDetail) {
// has author info already
if len(detail.Author) > 0 {
return
@ -1046,14 +1020,14 @@ func (ra *RepositoryAPI) VulnerabilityDetails() {
ra.ServeJSON()
}
func getSignatures(username, repository string) (map[string][]notary.Target, error) {
func getSignatures(username, repository string) (map[string][]notarymodel.Target, error) {
targets, err := notary.GetInternalTargets(config.InternalNotaryEndpoint(),
username, repository)
if err != nil {
return nil, err
}
signatures := map[string][]notary.Target{}
signatures := map[string][]notarymodel.Target{}
for _, tgt := range targets {
digest, err := notary.DigestFromTarget(tgt)
if err != nil {

10
src/core/api/repository_test.go
View File

@ -96,7 +96,7 @@ func TestGetReposTags(t *testing.T) {
t.Errorf("failed to get tags of repository %s: %v", repository, err)
} else {
assert.Equal(int(200), code, "httpStatusCode should be 200")
if tg, ok := tags.([]TagResp); ok {
if tg, ok := tags.([]models.TagResp); ok {
assert.Equal(1, len(tg), fmt.Sprintf("there should be only one tag, but now %v", tg))
assert.Equal(tg[0].Name, "latest", "the tag should be latest")
} else {
@ -207,19 +207,19 @@ func TestGetReposTop(t *testing.T) {
func TestPopulateAuthor(t *testing.T) {
author := "author"
detail := &tagDetail{
detail := &models.TagDetail{
Author: author,
}
populateAuthor(detail)
assert.Equal(t, author, detail.Author)
detail = &tagDetail{}
detail = &models.TagDetail{}
populateAuthor(detail)
assert.Equal(t, "", detail.Author)
maintainer := "maintainer"
detail = &tagDetail{
Config: &cfg{
detail = &models.TagDetail{
Config: &models.TagCfg{
Labels: map[string]string{
"Maintainer": maintainer,
},

5
src/pkg/clients/core/client.go
View File

@ -18,10 +18,11 @@ import (
"fmt"
"net/http"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/chartserver"
chttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/core/api"
)
// Client defines the methods that a core client should implement
@ -34,7 +35,7 @@ type Client interface {
// ImageClient defines the methods that an image client should implement
type ImageClient interface {
ListAllImages(project, repository string) ([]*api.TagResp, error)
ListAllImages(project, repository string) ([]*models.TagResp, error)
DeleteImage(project, repository, tag string) error
}

6
src/pkg/clients/core/image.go
View File

@ -17,12 +17,12 @@ package core
import (
"fmt"
"github.com/goharbor/harbor/src/core/api"
"github.com/goharbor/harbor/src/common/models"
)
func (c *client) ListAllImages(project, repository string) ([]*api.TagResp, error) {
func (c *client) ListAllImages(project, repository string) ([]*models.TagResp, error) {
url := c.buildURL(fmt.Sprintf("/api/repositories/%s/%s/tags", project, repository))
var images []*api.TagResp
var images []*models.TagResp
if err := c.httpclient.GetAndIteratePagination(url, &images); err != nil {
return nil, err
}

12
src/pkg/retention/client_test.go
View File

@ -18,8 +18,8 @@ import (
"testing"
"github.com/goharbor/harbor/src/chartserver"
"github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/core/api"
jmodels "github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/retention/res"
"github.com/goharbor/harbor/src/testing/clients"
@ -34,10 +34,10 @@ type fakeCoreClient struct {
clients.DumbCoreClient
}
func (f *fakeCoreClient) ListAllImages(project, repository string) ([]*api.TagResp, error) {
image := &api.TagResp{}
func (f *fakeCoreClient) ListAllImages(project, repository string) ([]*models.TagResp, error) {
image := &models.TagResp{}
image.Name = "latest"
return []*api.TagResp{image}, nil
return []*models.TagResp{image}, nil
}
func (f *fakeCoreClient) ListAllCharts(project, repository string) ([]*chartserver.ChartVersion, error) {
@ -53,7 +53,7 @@ func (f *fakeCoreClient) ListAllCharts(project, repository string) ([]*chartserv
type fakeJobserviceClient struct{}
func (f *fakeJobserviceClient) SubmitJob(*models.JobData) (string, error) {
func (f *fakeJobserviceClient) SubmitJob(*jmodels.JobData) (string, error) {
return "1", nil
}
func (f *fakeJobserviceClient) GetJobLog(uuid string) ([]byte, error) {

8
src/pkg/retention/launcher_test.go
View File

@ -216,15 +216,15 @@ func (l *launchTestSuite) TestLaunch() {
ScopeSelectors: map[string][]*rule.Selector{
"project": {
{
Kind: "regularExpression",
Decoration: "matches",
Kind: "doublestar",
Decoration: "nsMatches",
Pattern: "**",
},
},
"repository": {
{
Kind: "regularExpression",
Decoration: "matches",
Kind: "doublestar",
Decoration: "repoMatches",
Pattern: "**",
},
},

2
src/pkg/retention/policy/alg/or/processor_test.go
View File

@ -102,7 +102,7 @@ func (suite *ProcessorTestSuite) TearDownSuite() {}
func (suite *ProcessorTestSuite) TestProcess() {
results, err := suite.p.Process(suite.all)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), 2, len(results))
assert.Equal(suite.T(), 1, len(results))
assert.Condition(suite.T(), func() bool {
for _, r := range results {
if r.Error != nil {

18
src/pkg/retention/res/selectors/doublestar/selector.go
View File

@ -31,6 +31,10 @@ const (
RepoMatches = "repoMatches"
// RepoExcludes represents repository excludes [pattern]
RepoExcludes = "repoExcludes"
// NSMatches represents namespace matches [pattern]
NSMatches = "nsMatches"
// NSExcludes represents namespace excludes [pattern]
NSExcludes = "nsExcludes"
)
// selector for regular expression
@ -59,6 +63,11 @@ func (s *selector) Select(artifacts []*res.Candidate) (selected []*res.Candidate
case RepoExcludes:
value = art.Repository
excludes = true
case NSMatches:
value = art.Namespace
case NSExcludes:
value = art.Namespace
excludes = true
}
if len(value) > 0 {
@ -95,5 +104,12 @@ func match(pattern, str string) (bool, error) {
func init() {
// Register doublestar selector
selectors.Register(Kind, []string{Matches, Excludes}, New)
selectors.Register(Kind, []string{
Matches,
Excludes,
RepoMatches,
RepoExcludes,
NSMatches,
NSExcludes,
}, New)
}

62
src/pkg/retention/res/selectors/doublestar/selector_test.go
View File

@ -51,8 +51,8 @@ func (suite *RegExpSelectorTestSuite) SetupSuite() {
Labels: []string{"label1", "label2", "label3"},
},
{
NamespaceID: 1,
Namespace: "library",
NamespaceID: 2,
Namespace: "retention",
Repository: "redis",
Tag: "4.0",
Kind: res.Image,
@ -62,8 +62,8 @@ func (suite *RegExpSelectorTestSuite) SetupSuite() {
Labels: []string{"label1", "label4", "label5"},
},
{
NamespaceID: 1,
Namespace: "library",
NamespaceID: 2,
Namespace: "retention",
Repository: "redis",
Tag: "4.1",
Kind: res.Image,
@ -180,6 +180,60 @@ func (suite *RegExpSelectorTestSuite) TestRepoExcludes() {
})
}
// TestNSMatches tests the namespace `matches` case
func (suite *RegExpSelectorTestSuite) TestNSMatches() {
repoMatches := &selector{
decoration: NSMatches,
pattern: "{library}",
}
selected, err := repoMatches.Select(suite.artifacts)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), 1, len(selected))
assert.Condition(suite.T(), func() bool {
return expect([]string{"harbor:latest"}, selected)
})
repoMatches2 := &selector{
decoration: RepoMatches,
pattern: "re*",
}
selected, err = repoMatches2.Select(suite.artifacts)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), 2, len(selected))
assert.Condition(suite.T(), func() bool {
return expect([]string{"redis:4.0", "redis:4.1"}, selected)
})
}
// TestNSExcludes tests the namespace `excludes` case
func (suite *RegExpSelectorTestSuite) TestNSExcludes() {
repoExcludes := &selector{
decoration: NSExcludes,
pattern: "{library}",
}
selected, err := repoExcludes.Select(suite.artifacts)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), 2, len(selected))
assert.Condition(suite.T(), func() bool {
return expect([]string{"redis:4.0", "redis:4.1"}, selected)
})
repoExcludes2 := &selector{
decoration: NSExcludes,
pattern: "re*",
}
selected, err = repoExcludes2.Select(suite.artifacts)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), 1, len(selected))
assert.Condition(suite.T(), func() bool {
return expect([]string{"harbor:latest"}, selected)
})
}
// Check whether the returned result matched the expected ones (only check repo:tag)
func expect(expected []string, candidates []*res.Candidate) bool {
hash := make(map[string]bool)

4
src/testing/clients/dumb_core_client.go
View File

@ -16,7 +16,7 @@ package clients
import (
"github.com/goharbor/harbor/src/chartserver"
"github.com/goharbor/harbor/src/core/api"
"github.com/goharbor/harbor/src/common/models"
)
// DumbCoreClient provides an empty implement for pkg/clients/core.Client
@ -24,7 +24,7 @@ import (
type DumbCoreClient struct{}
// ListAllImages ...
func (d *DumbCoreClient) ListAllImages(project, repository string) ([]*api.TagResp, error) {
func (d *DumbCoreClient) ListAllImages(project, repository string) ([]*models.TagResp, error) {
return nil, nil
}