Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-18 14:47:38 +01:00
Code Issues Projects Releases Wiki Activity

Implement tag/artifact manager and artifact controller

Browse Source 
1. Implement tag/artifact manager
2. Implement artifact controller
3. Onboard the artifact when pushing artifacts

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2019-12-20 09:24:57 +08:00
parent 82adac43b0
commit 400a47a5c5
42 changed files with 3154 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

277
src/api/artifact/controller.go
View File

@ -16,33 +16,42 @@ package artifact
import (
"context"
"github.com/goharbor/harbor/src/common/models"
"fmt"
"github.com/goharbor/harbor/src/common/utils/log"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/repository"
"github.com/goharbor/harbor/src/pkg/tag"
tm "github.com/goharbor/harbor/src/pkg/tag/model/tag"
"time"
)
var (
// Ctl is a global artifact controller instance
Ctl = NewController()
Ctl = NewController(repository.Mgr, artifact.Mgr, tag.Mgr)
)
// Controller defines the operations related with artifacts and tags
type Controller interface {
// Ensure the artifact specified by the digest exists under the repository,
// creates it if it doesn't exist. If tags are provided, ensure they exist
// and are attached to the artifact. If the tags don't exist, create them first
Ensure(ctx context.Context, repository *models.RepoRecord, digest string, tags ...string) (err error)
// List artifacts according to the query and option
// and are attached to the artifact. If the tags don't exist, create them first.
// The "created" will be set as true when the artifact is created
Ensure(ctx context.Context, repositoryID int64, digest string, tags ...string) (created bool, id int64, err error)
// List artifacts according to the query, specify the properties returned with option
List(ctx context.Context, query *q.Query, option *Option) (total int64, artifacts []*Artifact, err error)
// Get the artifact specified by ID
// Get the artifact specified by ID, specify the properties returned with option
Get(ctx context.Context, id int64, option *Option) (artifact *Artifact, err error)
// Delete the artifact specified by ID
// Delete the artifact specified by ID. All tags attached to the artifact are deleted as well
Delete(ctx context.Context, id int64) (err error)
// DeleteTag deletes the tag specified by ID
DeleteTag(ctx context.Context, id int64) (err error)
// UpdatePullTime updates the pull time for the artifact. If the tag is provides, update the pull
// Tags returns the tags according to the query, specify the properties returned with option
Tags(ctx context.Context, query *q.Query, option *TagOption) (total int64, tags []*Tag, err error)
// DeleteTag deletes the tag specified by tagID
DeleteTag(ctx context.Context, tagID int64) (err error)
// UpdatePullTime updates the pull time for the artifact. If the tagID is provides, update the pull
// time of the tag as well
UpdatePullTime(ctx context.Context, artifactID int64, tag string, time time.Time) (err error)
UpdatePullTime(ctx context.Context, artifactID int64, tagID int64, time time.Time) (err error)
// GetSubResource returns the sub resource of the artifact
// The sub resource is different according to the artifact type:
// build history for image; values.yaml, readme and dependencies for chart, etc
@ -54,10 +63,246 @@ type Controller interface {
}
// NewController creates an instance of the default artifact controller
func NewController() Controller {
// TODO implement
return nil
func NewController(repoMgr repository.Manager, artMgr artifact.Manager, tagMgr tag.Manager) Controller {
return &controller{
repoMgr: repoMgr,
artMgr: artMgr,
tagMgr: tagMgr,
}
}
// As a redis lock is applied during the artifact pushing, we do not to handle the concurrent issues
// for artifacts and tags
// TODO handle concurrency, the redis lock doesn't cover all cases
// TODO As a redis lock is applied during the artifact pushing, we do not to handle the concurrent issues
// for artifacts and tags？？
type controller struct {
repoMgr repository.Manager
artMgr artifact.Manager
tagMgr tag.Manager
}
func (c *controller) Ensure(ctx context.Context, repositoryID int64, digest string, tags ...string) (bool, int64, error) {
created, id, err := c.ensureArtifact(ctx, repositoryID, digest)
if err != nil {
return false, 0, err
}
for _, tag := range tags {
if err = c.ensureTag(ctx, repositoryID, id, tag); err != nil {
return false, 0, err
}
}
return created, id, nil
}
// ensure the artifact exists under the repository, create it if doesn't exist.
func (c *controller) ensureArtifact(ctx context.Context, repositoryID int64, digest string) (bool, int64, error) {
query := &q.Query{
Keywords: map[string]interface{}{
"repository_id": repositoryID,
"digest": digest,
},
}
_, artifacts, err := c.artMgr.List(ctx, query)
if err != nil {
return false, 0, err
}
// the artifact already exists under the repository, return directly
if len(artifacts) > 0 {
return false, artifacts[0].ID, nil
}
// the artifact doesn't exist under the repository, create it first
repository, err := c.repoMgr.Get(ctx, repositoryID)
if err != nil {
return false, 0, err
}
artifact := &artifact.Artifact{
ProjectID: repository.ProjectID,
RepositoryID: repositoryID,
Digest: digest,
PushTime: time.Now(),
}
// abstract the specific information for the artifact
c.abstract(ctx, artifact)
// create it
id, err := c.artMgr.Create(ctx, artifact)
if err != nil {
// if got conflict error, try to get the artifact again
if ierror.IsConflictErr(err) {
_, artifacts, err = c.artMgr.List(ctx, query)
if err != nil {
return false, 0, err
}
if len(artifacts) > 0 {
return false, artifacts[0].ID, nil
}
}
return false, 0, err
}
return true, id, nil
}
func (c *controller) ensureTag(ctx context.Context, repositoryID, artifactID int64, name string) error {
query := &q.Query{
Keywords: map[string]interface{}{
"repository_id": repositoryID,
"name": name,
},
}
_, tags, err := c.tagMgr.List(ctx, query)
if err != nil {
return err
}
// the tag already exists under the repository
if len(tags) > 0 {
tag := tags[0]
// the tag already exists under the repository and is attached to the artifact, return directly
if tag.ArtifactID == artifactID {
return nil
}
// the tag exists under the repository, but it is attached to other artifact
// update it to point to the provided artifact
tag.ArtifactID = artifactID
tag.PushTime = time.Now()
return c.tagMgr.Update(ctx, tag, "ArtifactID", "PushTime")
}
// the tag doesn't exist under the repository, create it
_, err = c.tagMgr.Create(ctx, &tm.Tag{
RepositoryID: repositoryID,
ArtifactID: artifactID,
Name: name,
PushTime: time.Now(),
})
// ignore the conflict error
if err != nil && ierror.IsConflictErr(err) {
return nil
}
return err
}
func (c *controller) List(ctx context.Context, query *q.Query, option *Option) (int64, []*Artifact, error) {
total, arts, err := c.artMgr.List(ctx, query)
if err != nil {
return 0, nil, err
}
var artifacts []*Artifact
for _, art := range arts {
artifacts = append(artifacts, c.assembleArtifact(ctx, art, option))
}
return total, artifacts, nil
}
func (c *controller) Get(ctx context.Context, id int64, option *Option) (*Artifact, error) {
art, err := c.artMgr.Get(ctx, id)
if err != nil {
return nil, err
}
return c.assembleArtifact(ctx, art, option), nil
}
func (c *controller) Delete(ctx context.Context, id int64) error {
// delete artifact first in case the artifact is referenced by other artifact
if err := c.artMgr.Delete(ctx, id); err != nil {
return err
}
// delete all tags that attached to the artifact
_, tags, err := c.tagMgr.List(ctx, &q.Query{
Keywords: map[string]interface{}{
"artifact_id": id,
},
})
if err != nil {
return err
}
for _, tag := range tags {
if err = c.DeleteTag(ctx, tag.ID); err != nil {
return err
}
}
// TODO fire delete artifact event
return nil
}
func (c *controller) Tags(ctx context.Context, query *q.Query, option *TagOption) (int64, []*Tag, error) {
total, tgs, err := c.tagMgr.List(ctx, query)
if err != nil {
return 0, nil, err
}
var tags []*Tag
for _, tg := range tgs {
tags = append(tags, c.assembleTag(ctx, tg, option))
}
return total, tags, nil
}
func (c *controller) DeleteTag(ctx context.Context, tagID int64) error {
// immutable checking is covered in middleware
// TODO check signature
// TODO delete label
// TODO fire delete tag event
return c.tagMgr.Delete(ctx, tagID)
}
func (c *controller) UpdatePullTime(ctx context.Context, artifactID int64, tagID int64, time time.Time) error {
tag, err := c.tagMgr.Get(ctx, tagID)
if err != nil {
return err
}
if tag.ArtifactID != artifactID {
return fmt.Errorf("tag %d isn't attached to artifact %d", tagID, artifactID)
}
if err := c.artMgr.UpdatePullTime(ctx, artifactID, time); err != nil {
return err
}
return c.tagMgr.Update(ctx, &tm.Tag{
ID: tagID,
}, "PullTime")
}
func (c *controller) GetSubResource(ctx context.Context, artifactID int64, resource string) (*Resource, error) {
// TODO implement
return nil, nil
}
// assemble several part into a single artifact
func (c *controller) assembleArtifact(ctx context.Context, art *artifact.Artifact, option *Option) *Artifact {
artifact := &Artifact{
Artifact: *art,
}
if option == nil {
return artifact
}
// populate tags
if option.WithTag {
_, tgs, err := c.tagMgr.List(ctx, &q.Query{
Keywords: map[string]interface{}{
"artifact_id": artifact.ID,
},
})
if err == nil {
// assemble tags
for _, tg := range tgs {
artifact.Tags = append(artifact.Tags, c.assembleTag(ctx, tg, option.TagOption))
}
} else {
log.Errorf("failed to list tag of artifact %d: %v", artifact.ID, err)
}
}
// TODO populate other properties: scan, signature etc.
return artifact
}
// assemble several part into a single tag
func (c *controller) assembleTag(ctx context.Context, tag *tm.Tag, option *TagOption) *Tag {
t := &Tag{
Tag: *tag,
}
if option == nil {
return t
}
// TODO populate label, signature, immutable status for tag
return t
}
func (c *controller) abstract(ctx context.Context, artifact *artifact.Artifact) {
// TODO abstract the specific info for the artifact
// handler references
}

310
src/api/artifact/controller_test.go Normal file
View File

@ -0,0 +1,310 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package artifact
import (
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/tag/model/tag"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/stretchr/testify/suite"
"testing"
"time"
)
type controllerTestSuite struct {
suite.Suite
ctl *controller
repoMgr *htesting.FakeRepositoryManager
artMgr *htesting.FakeArtifactManager
tagMgr *htesting.FakeTagManager
}
func (c *controllerTestSuite) SetupTest() {
c.repoMgr = &htesting.FakeRepositoryManager{}
c.artMgr = &htesting.FakeArtifactManager{}
c.tagMgr = &htesting.FakeTagManager{}
c.ctl = &controller{
repoMgr: c.repoMgr,
artMgr: c.artMgr,
tagMgr: c.tagMgr,
}
}
func (c *controllerTestSuite) TestAssembleTag() {
tg := &tag.Tag{
ID: 1,
RepositoryID: 1,
ArtifactID: 1,
Name: "latest",
PushTime: time.Now(),
PullTime: time.Now(),
}
option := &TagOption{
WithLabel: true,
WithImmutableStatus: true,
}
tag := c.ctl.assembleTag(nil, tg, option)
c.Require().NotNil(tag)
c.Equal(tag.ID, tg.ID)
// TODO check other fields of option
}
func (c *controllerTestSuite) TestAssembleArtifact() {
art := &artifact.Artifact{
ID: 1,
}
option := &Option{
WithTag: true,
TagOption: &TagOption{
WithLabel: false,
WithImmutableStatus: false,
},
WithScanResult: true,
WithSignature: true,
}
tg := &tag.Tag{
ID: 1,
RepositoryID: 1,
ArtifactID: 1,
Name: "latest",
PushTime: time.Now(),
PullTime: time.Now(),
}
c.tagMgr.On("List").Return(1, []*tag.Tag{tg}, nil)
artifact := c.ctl.assembleArtifact(nil, art, option)
c.Require().NotNil(artifact)
c.tagMgr.AssertExpectations(c.T())
c.Equal(art.ID, artifact.ID)
c.Contains(artifact.Tags, &Tag{Tag: *tg})
// TODO check other fields of option
}
func (c *controllerTestSuite) TestAbstract() {
// TODO add test case
}
func (c *controllerTestSuite) TestEnsureArtifact() {
digest := "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180"
// the artifact already exists
c.artMgr.On("List").Return(1, []*artifact.Artifact{
{
ID: 1,
},
}, nil)
created, id, err := c.ctl.ensureArtifact(nil, 1, digest)
c.Require().Nil(err)
c.repoMgr.AssertExpectations(c.T())
c.artMgr.AssertExpectations(c.T())
c.False(created)
c.Equal(int64(1), id)
// reset the mock
c.SetupTest()
c.repoMgr.On("Get").Return(&models.RepoRecord{
ProjectID: 1,
}, nil)
// the artifact doesn't exist
c.artMgr.On("List").Return(1, []*artifact.Artifact{}, nil)
c.artMgr.On("Create").Return(1, nil)
created, id, err = c.ctl.ensureArtifact(nil, 1, digest)
c.Require().Nil(err)
c.repoMgr.AssertExpectations(c.T())
c.artMgr.AssertExpectations(c.T())
c.True(created)
c.Equal(int64(1), id)
}
func (c *controllerTestSuite) TestEnsureTag() {
// the tag already exists under the repository and is attached to the artifact
c.tagMgr.On("List").Return(1, []*tag.Tag{
{
ID: 1,
RepositoryID: 1,
ArtifactID: 1,
Name: "latest",
},
}, nil)
err := c.ctl.ensureTag(nil, 1, 1, "latest")
c.Require().Nil(err)
c.tagMgr.AssertExpectations(c.T())
// reset the mock
c.SetupTest()
// the tag exists under the repository, but it is attached to other artifact
c.tagMgr.On("List").Return(1, []*tag.Tag{
{
ID: 1,
RepositoryID: 1,
ArtifactID: 2,
Name: "latest",
},
}, nil)
c.tagMgr.On("Update").Return(nil)
err = c.ctl.ensureTag(nil, 1, 1, "latest")
c.Require().Nil(err)
c.tagMgr.AssertExpectations(c.T())
// reset the mock
c.SetupTest()
// the tag doesn't exist under the repository, create it
c.tagMgr.On("List").Return(1, []*tag.Tag{}, nil)
c.tagMgr.On("Create").Return(1, nil)
err = c.ctl.ensureTag(nil, 1, 1, "latest")
c.Require().Nil(err)
c.tagMgr.AssertExpectations(c.T())
}
func (c *controllerTestSuite) TestEnsure() {
digest := "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180"
// both the artifact and the tag don't exist
c.repoMgr.On("Get").Return(&models.RepoRecord{
ProjectID: 1,
}, nil)
c.artMgr.On("List").Return(1, []*artifact.Artifact{}, nil)
c.artMgr.On("Create").Return(1, nil)
c.tagMgr.On("List").Return(1, []*tag.Tag{}, nil)
c.tagMgr.On("Create").Return(1, nil)
_, id, err := c.ctl.Ensure(nil, 1, digest, "latest")
c.Require().Nil(err)
c.repoMgr.AssertExpectations(c.T())
c.artMgr.AssertExpectations(c.T())
c.tagMgr.AssertExpectations(c.T())
c.Equal(int64(1), id)
}
func (c *controllerTestSuite) TestList() {
query := &q.Query{}
option := &Option{
WithTag: true,
WithScanResult: true,
WithSignature: true,
}
c.artMgr.On("List").Return(1, []*artifact.Artifact{
{
ID: 1,
RepositoryID: 1,
},
}, nil)
c.tagMgr.On("List").Return(1, []*tag.Tag{
{
ID: 1,
RepositoryID: 1,
ArtifactID: 1,
Name: "latest",
},
}, nil)
total, artifacts, err := c.ctl.List(nil, query, option)
c.Require().Nil(err)
c.artMgr.AssertExpectations(c.T())
c.tagMgr.AssertExpectations(c.T())
c.Equal(int64(1), total)
c.Require().Len(artifacts, 1)
c.Equal(int64(1), artifacts[0].ID)
c.Require().Len(artifacts[0].Tags, 1)
c.Equal(int64(1), artifacts[0].Tags[0].ID)
}
func (c *controllerTestSuite) TestGet() {
c.artMgr.On("Get").Return(&artifact.Artifact{
ID: 1,
RepositoryID: 1,
}, nil)
art, err := c.ctl.Get(nil, 1, nil)
c.Require().Nil(err)
c.artMgr.AssertExpectations(c.T())
c.Require().NotNil(art)
c.Equal(int64(1), art.ID)
}
func (c *controllerTestSuite) TestDelete() {
c.artMgr.On("Delete").Return(nil)
c.tagMgr.On("List").Return(0, []*tag.Tag{
{
ID: 1,
},
}, nil)
c.tagMgr.On("Delete").Return(nil)
err := c.ctl.Delete(nil, 1)
c.Require().Nil(err)
c.artMgr.AssertExpectations(c.T())
c.tagMgr.AssertExpectations(c.T())
}
func (c *controllerTestSuite) TestTags() {
c.tagMgr.On("List").Return(1, []*tag.Tag{
{
ID: 1,
RepositoryID: 1,
ArtifactID: 1,
Name: "latest",
},
}, nil)
total, tags, err := c.ctl.Tags(nil, nil, nil)
c.Require().Nil(err)
c.Equal(int64(1), total)
c.Len(tags, 1)
c.tagMgr.AssertExpectations(c.T())
// TODO check other properties: label, etc
}
func (c *controllerTestSuite) TestDeleteTag() {
c.tagMgr.On("Delete").Return(nil)
err := c.ctl.DeleteTag(nil, 1)
c.Require().Nil(err)
c.tagMgr.AssertExpectations(c.T())
}
func (c *controllerTestSuite) TestUpdatePullTime() {
// artifact ID and tag ID matches
c.tagMgr.On("Get").Return(&tag.Tag{
ID: 1,
ArtifactID: 1,
}, nil)
c.artMgr.On("UpdatePullTime").Return(nil)
c.tagMgr.On("Update").Return(nil)
err := c.ctl.UpdatePullTime(nil, 1, 1, time.Now())
c.Require().Nil(err)
c.artMgr.AssertExpectations(c.T())
c.tagMgr.AssertExpectations(c.T())
// reset the mock
c.SetupTest()
// artifact ID and tag ID doesn't match
c.tagMgr.On("Get").Return(&tag.Tag{
ID: 1,
ArtifactID: 2,
}, nil)
err = c.ctl.UpdatePullTime(nil, 1, 1, time.Now())
c.Require().NotNil(err)
c.tagMgr.AssertExpectations(c.T())
}
func (c *controllerTestSuite) TestGetSubResource() {
// TODO
}
func TestControllerTestSuite(t *testing.T) {
suite.Run(t, &controllerTestSuite{})
}

20
src/api/artifact/model.go
View File

@ -23,9 +23,15 @@ import (
// TODO reuse the model generated by swagger
type Artifact struct {
artifact.Artifact
Tags []*tag.Tag // the list of tags that attached to the artifact
Tags []*Tag // the list of tags that attached to the artifact
SubResourceLinks map[string][]*ResourceLink // the resource link for build history(image), values.yaml(chart), dependency(chart), etc
// TODO add other attrs: signature, scan result, label, etc
// TODO add other attrs: signature, scan result, etc
}
// Tag is the overall view of tag
type Tag struct {
tag.Tag
// TODO add other attrs: signature, label, immutable status, etc
}
// Resource defines the specific resource of different artifacts: build history for image, values.yaml for chart, etc
@ -43,9 +49,15 @@ type ResourceLink struct {
// Option is used to specify the properties returned when listing/getting artifacts
type Option struct {
WithTag bool
WithLabel bool
TagOption *TagOption // only works when WithTag is set to true
WithScanResult bool
WithSignature bool
WithSignature bool // TODO move it to TagOption?
}
// TagOption is used to specify the properties returned when listing/getting tags
type TagOption struct {
WithLabel bool
WithImmutableStatus bool
}
// TODO move this to GC controller?

89
src/api/repository/controller.go Normal file
View File

@ -0,0 +1,89 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package repository
import (
"context"
"github.com/goharbor/harbor/src/common/models"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/repository"
)
var (
// Ctl is a global repository controller instance
Ctl = NewController(repository.Mgr)
)
// Controller defines the operations related with repositories
type Controller interface {
// Ensure the repository specified by the "name" exists under the project,
// creates it if it doesn't exist. The "name" should contain the namespace part.
// The "created" will be set as true when the repository is created
Ensure(ctx context.Context, projectID int64, name string) (created bool, id int64, err error)
// Get the repository specified by ID
Get(ctx context.Context, id int64) (repository *models.RepoRecord, err error)
}
// NewController creates an instance of the default repository controller
func NewController(repoMgr repository.Manager) Controller {
return &controller{
repoMgr: repoMgr,
}
}
type controller struct {
repoMgr repository.Manager
}
func (c *controller) Ensure(ctx context.Context, projectID int64, name string) (bool, int64, error) {
query := &q.Query{
Keywords: map[string]interface{}{
"name": name,
},
}
_, repositories, err := c.repoMgr.List(ctx, query)
if err != nil {
return false, 0, err
}
// the repository already exists, return directly
if len(repositories) > 0 {
return false, repositories[0].RepositoryID, nil
}
// the repository doesn't exist, create it first
id, err := c.repoMgr.Create(ctx, &models.RepoRecord{
ProjectID: projectID,
Name: name,
})
if err != nil {
// if got conflict error, try to get again
if ierror.IsConflictErr(err) {
_, repositories, err = c.repoMgr.List(ctx, query)
if err != nil {
return false, 0, err
}
if len(repositories) > 0 {
return false, repositories[0].RepositoryID, nil
}
}
return false, 0, err
}
return true, id, nil
}
func (c *controller) Get(ctx context.Context, id int64) (*models.RepoRecord, error) {
return c.repoMgr.Get(ctx, id)
}

77
src/api/repository/controller_test.go Normal file
View File

@ -0,0 +1,77 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package repository
import (
"github.com/goharbor/harbor/src/common/models"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/stretchr/testify/suite"
"testing"
)
type controllerTestSuite struct {
suite.Suite
ctl *controller
repoMgr *htesting.FakeRepositoryManager
}
func (c *controllerTestSuite) SetupTest() {
c.repoMgr = &htesting.FakeRepositoryManager{}
c.ctl = &controller{
repoMgr: c.repoMgr,
}
}
func (c *controllerTestSuite) TestEnsure() {
// already exists
c.repoMgr.On("List").Return(0, []*models.RepoRecord{
{
RepositoryID: 1,
ProjectID: 1,
Name: "library/hello-world",
},
}, nil)
created, id, err := c.ctl.Ensure(nil, 1, "library/hello-world")
c.Require().Nil(err)
c.repoMgr.AssertExpectations(c.T())
c.False(created)
c.Equal(int64(1), id)
// reset the mock
c.SetupTest()
// doesn't exist
c.repoMgr.On("List").Return(0, []*models.RepoRecord{}, nil)
c.repoMgr.On("Create").Return(1, nil)
created, id, err = c.ctl.Ensure(nil, 1, "library/hello-world")
c.Require().Nil(err)
c.repoMgr.AssertExpectations(c.T())
c.True(created)
c.Equal(int64(1), id)
}
func (c *controllerTestSuite) TestGet() {
c.repoMgr.On("Get").Return(&models.RepoRecord{
RepositoryID: 1,
}, nil)
repository, err := c.ctl.Get(nil, 1)
c.Require().Nil(err)
c.repoMgr.AssertExpectations(c.T())
c.Equal(int64(1), repository.RepositoryID)
}
func TestControllerTestSuite(t *testing.T) {
suite.Run(t, &controllerTestSuite{})
}

35
src/common/api/base.go
View File

@ -25,7 +25,7 @@ import (
"github.com/astaxie/beego/validation"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils/log"
internal_errors "github.com/goharbor/harbor/src/internal/error"
serror "github.com/goharbor/harbor/src/server/error"
)
const (
@ -262,35 +262,6 @@ func (b *BaseAPI) SendStatusServiceUnavailableError(err error) {
// ]
// }
func (b *BaseAPI) SendError(err error) {
var statusCode int
var send error
var e *internal_errors.Error
if errors.As(err, &e) {
code := e.Code
statusCode = b.getStatusCode(code)
if statusCode == 0 {
statusCode = http.StatusInternalServerError
}
send = e
} else {
statusCode = http.StatusInternalServerError
send = internal_errors.UnknownError(err)
}
b.RenderError(statusCode, internal_errors.NewErrs(send).Error())
}
func (b *BaseAPI) getStatusCode(code string) int {
statusCodeMap := map[string]int{
internal_errors.NotFoundCode: http.StatusNotFound,
internal_errors.ConflictCode: http.StatusConflict,
internal_errors.UnAuthorizedCode: http.StatusUnauthorized,
internal_errors.BadRequestCode: http.StatusBadRequest,
internal_errors.ForbiddenCode: http.StatusForbidden,
internal_errors.PreconditionCode: http.StatusPreconditionFailed,
internal_errors.GeneralCode: http.StatusInternalServerError,
}
return statusCodeMap[code]
statusCode, payload := serror.APIError(err)
b.RenderError(statusCode, payload)
}

14
src/core/api/base.go
View File

@ -45,7 +45,6 @@ const (
// the managers/controllers used globally
var (
projectMgr project.Manager
repositoryMgr repository.Manager
retentionScheduler scheduler.Scheduler
retentionMgr retention.Manager
retentionLauncher retention.Launcher
@ -193,16 +192,13 @@ func Init() error {
// init project manager
initProjectManager()
// init repository manager
initRepositoryManager()
initRetentionScheduler()
retentionMgr = retention.NewManager()
retentionLauncher = retention.NewLauncher(projectMgr, repositoryMgr, retentionMgr)
retentionLauncher = retention.NewLauncher(projectMgr, repository.Mgr, retentionMgr)
retentionController = retention.NewAPIController(retentionMgr, projectMgr, repositoryMgr, retentionScheduler, retentionLauncher)
retentionController = retention.NewAPIController(retentionMgr, projectMgr, repository.Mgr, retentionScheduler, retentionLauncher)
callbackFun := func(p interface{}) error {
str, ok := p.(string)
@ -237,11 +233,7 @@ func initChartController() error {
}
func initProjectManager() {
projectMgr = project.New()
}
func initRepositoryManager() {
repositoryMgr = repository.New(projectMgr, chartController)
projectMgr = project.Mgr
}
func initRetentionScheduler() {

8
src/core/router.go
View File

@ -25,6 +25,8 @@ import (
"github.com/goharbor/harbor/src/core/service/notifications/registry"
"github.com/goharbor/harbor/src/core/service/notifications/scheduler"
"github.com/goharbor/harbor/src/core/service/token"
reg "github.com/goharbor/harbor/src/server/registry"
"net/url"
)
func initRouters() {
@ -158,6 +160,12 @@ func initRouters() {
beego.Router("/api/projects/:pid([0-9]+)/immutabletagrules", &api.ImmutableTagRuleAPI{}, "get:List;post:Post")
beego.Router("/api/projects/:pid([0-9]+)/immutabletagrules/:id([0-9]+)", &api.ImmutableTagRuleAPI{})
// TODO remove
regURL, _ := config.RegistryURL()
url, _ := url.Parse(regURL)
registryHandler := reg.New(url)
_ = registryHandler
// beego.Handler("/v2/*", registryHandler)
beego.Router("/v2/*", &controllers.RegistryProxy{}, "*:Handle")
// APIs for chart repository

41
src/internal/error/errors.go
View File

@ -2,9 +2,9 @@ package error
import (
"encoding/json"
"errors"
"fmt"
"github.com/goharbor/harbor/src/common/utils/log"
"strings"
)
// Error ...
@ -20,8 +20,8 @@ func (e *Error) Error() string {
}
// WithMessage ...
func (e *Error) WithMessage(msg string) *Error {
e.Message = msg
func (e *Error) WithMessage(format string, v ...interface{}) *Error {
e.Message = fmt.Sprintf(format, v...)
return e
}
@ -51,7 +51,7 @@ func (errs Errors) Error() string {
case *Error:
err = e.(*Error)
default:
err = UnknownError(e).WithMessage(err.Error())
err = UnknownError(e).WithMessage(e.Error())
}
tmpErrs.Errors = append(tmpErrs.Errors, *err.(*Error))
}
@ -84,7 +84,7 @@ const (
// BadRequestCode ...
BadRequestCode = "BAD_REQUEST"
// ForbiddenCode ...
ForbiddenCode = "FORBIDDER"
ForbiddenCode = "FORBIDDEN"
// PreconditionCode ...
PreconditionCode = "PRECONDITION"
// GeneralCode ...
@ -93,13 +93,15 @@ const (
// New ...
func New(err error) *Error {
if _, ok := err.(*Error); ok {
err = err.(*Error).Unwrap()
}
return &Error{
Cause: err,
Message: err.Error(),
e := &Error{}
if err != nil {
e.Cause = err
e.Message = err.Error()
if ee, ok := err.(*Error); ok {
e.Cause = ee
}
}
return e
}
// NotFoundError is error for the case of object not found
@ -129,7 +131,7 @@ func ForbiddenError(err error) *Error {
// PreconditionFailedError is error for the case of precondition failed
func PreconditionFailedError(err error) *Error {
return New(err).WithCode(PreconditionCode).WithMessage("preconfition")
return New(err).WithCode(PreconditionCode).WithMessage("precondition failed")
}
// UnknownError ...
@ -137,11 +139,16 @@ func UnknownError(err error) *Error {
return New(err).WithCode(GeneralCode).WithMessage("unknown")
}
// IsErr ...
// IsErr checks whether the err chain contains error matches the code
func IsErr(err error, code string) bool {
_, ok := err.(*Error)
if !ok {
return false
var e *Error
if errors.As(err, &e) {
return e.Code == code
}
return strings.Compare(err.(*Error).Code, code) == 0
return false
}
// IsConflictErr checks whether the err chain contains conflict error
func IsConflictErr(err error) bool {
return IsErr(err, ConflictCode)
}

51
src/internal/orm/error.go Normal file
View File

@ -0,0 +1,51 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package orm
import (
"errors"
"github.com/astaxie/beego/orm"
ierror "github.com/goharbor/harbor/src/internal/error"
"strings"
)
// IsNotFoundError checks whether the err is orm.ErrNoRows. If it it, wrap it
// as a src/internal/error.Error with not found error code
func IsNotFoundError(err error, messageFormat string, args ...interface{}) (*ierror.Error, bool) {
if errors.Is(err, orm.ErrNoRows) {
e := ierror.NotFoundError(err)
if len(messageFormat) > 0 {
e.WithMessage(messageFormat, args...)
}
return e, true
}
return nil, false
}
// IsConflictError checks whether the err is duplicate key error. If it it, wrap it
// as a src/internal/error.Error with conflict error code
func IsConflictError(err error, messageFormat string, args ...interface{}) (*ierror.Error, bool) {
if err == nil {
return nil, false
}
if strings.Contains(err.Error(), "duplicate key value violates unique constraint") {
e := ierror.ConflictError(err)
if len(messageFormat) > 0 {
e.WithMessage(messageFormat, args...)
}
return e, true
}
return nil, false
}

57
src/internal/orm/error_test.go Normal file
View File

@ -0,0 +1,57 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package orm
import (
"errors"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/internal/error"
"github.com/stretchr/testify/assert"
"testing"
)
func TestIsNotFoundError(t *testing.T) {
// nil error
_, ok := IsNotFoundError(nil, "")
assert.False(t, ok)
// common error
_, ok = IsNotFoundError(errors.New("common error"), "")
assert.False(t, ok)
// pass
message := "message"
e, ok := IsNotFoundError(orm.ErrNoRows, message)
assert.True(t, ok)
assert.Equal(t, error.NotFoundCode, e.Code)
assert.Equal(t, message, e.Message)
}
func TestIsConflictError(t *testing.T) {
// nil error
_, ok := IsConflictError(nil, "")
assert.False(t, ok)
// common error
_, ok = IsConflictError(errors.New("common error"), "")
assert.False(t, ok)
// pass
message := "message"
e, ok := IsConflictError(errors.New("duplicate key value violates unique constraint"), message)
assert.True(t, ok)
assert.Equal(t, error.ConflictCode, e.Code)
assert.Equal(t, message, e.Message)
}

46
src/internal/orm/query.go Normal file
View File

@ -0,0 +1,46 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package orm
import (
"context"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/pkg/q"
)
// QuerySetter generates the query setter according to the query
func QuerySetter(ctx context.Context, model interface{}, query *q.Query) orm.QuerySeter {
qs := GetOrmer(ctx).QueryTable(model)
if query == nil {
return qs
}
for k, v := range query.Keywords {
qs = qs.Filter(k, v)
}
if query.PageSize > 0 {
qs = qs.Limit(query.PageSize)
if query.PageNumber > 0 {
qs = qs.Offset(query.PageSize * (query.PageNumber - 1))
}
}
return qs
}
// GetOrmer returns an ormer
// TODO remove it after weiwei's PR merged
func GetOrmer(ctx context.Context) orm.Ormer {
return dao.GetOrmer()
}

77
src/internal/response_buffer.go Normal file
View File

@ -0,0 +1,77 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package internal
import (
"bytes"
"net/http"
)
// ResponseBuffer is a wrapper for the http.ResponseWriter to buffer the response data
type ResponseBuffer struct {
w http.ResponseWriter
code int
header http.Header
buffer bytes.Buffer
wroteHeader bool
}
// NewResponseBuffer creates a ResponseBuffer object
func NewResponseBuffer(w http.ResponseWriter) *ResponseBuffer {
return &ResponseBuffer{
w: w,
header: http.Header{},
buffer: bytes.Buffer{},
}
}
// WriteHeader writes the status code into the buffer without writing to the underlying response writer
func (r *ResponseBuffer) WriteHeader(statusCode int) {
if r.wroteHeader {
return
}
r.wroteHeader = true
r.code = statusCode
}
// Write writes the data into the buffer without writing to the underlying response writer
func (r *ResponseBuffer) Write(data []byte) (int, error) {
r.WriteHeader(http.StatusOK)
return r.buffer.Write(data)
}
// Header returns the header of the buffer
func (r *ResponseBuffer) Header() http.Header {
return r.header
}
// Flush the status code, header and data into the underlying response writer
func (r *ResponseBuffer) Flush() (int, error) {
header := r.w.Header()
for k, vs := range r.header {
for _, v := range vs {
header.Add(k, v)
}
}
if r.code > 0 {
r.w.WriteHeader(r.code)
}
return r.w.Write(r.buffer.Bytes())
}
// Success checks whether the status code is >= 200 & <= 399
func (r *ResponseBuffer) Success() bool {
return r.code >= 200 && r.code <= 399
}

86
src/internal/response_buffer_test.go Normal file
View File

@ -0,0 +1,86 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package internal
import (
"github.com/stretchr/testify/suite"
"net/http"
"net/http/httptest"
"testing"
)
type responseBufferTestSuite struct {
suite.Suite
recorder *httptest.ResponseRecorder
buffer *ResponseBuffer
}
func (r *responseBufferTestSuite) SetupTest() {
r.recorder = httptest.NewRecorder()
r.buffer = NewResponseBuffer(r.recorder)
}
func (r *responseBufferTestSuite) TestWriteHeader() {
// write once
r.buffer.WriteHeader(http.StatusInternalServerError)
r.Equal(http.StatusInternalServerError, r.buffer.code)
r.Equal(http.StatusOK, r.recorder.Code)
// write again
r.buffer.WriteHeader(http.StatusNotFound)
r.Equal(http.StatusInternalServerError, r.buffer.code)
r.Equal(http.StatusOK, r.recorder.Code)
}
func (r *responseBufferTestSuite) TestWrite() {
_, err := r.buffer.Write([]byte{'a'})
r.Require().Nil(err)
r.Equal([]byte{'a'}, r.buffer.buffer.Bytes())
r.Empty(r.recorder.Body.Bytes())
// try to write header after calling write
r.buffer.WriteHeader(http.StatusNotFound)
r.Equal(http.StatusOK, r.buffer.code)
}
func (r *responseBufferTestSuite) TestHeader() {
header := r.buffer.Header()
header.Add("k", "v")
r.Equal("v", r.buffer.header.Get("k"))
r.Empty(r.recorder.Header())
}
func (r *responseBufferTestSuite) TestFlush() {
r.buffer.WriteHeader(http.StatusOK)
_, err := r.buffer.Write([]byte{'a'})
r.Require().Nil(err)
_, err = r.buffer.Flush()
r.Require().Nil(err)
r.Equal(http.StatusOK, r.recorder.Code)
r.Equal([]byte{'a'}, r.recorder.Body.Bytes())
}
func (r *responseBufferTestSuite) TestSuccess() {
r.buffer.WriteHeader(http.StatusInternalServerError)
r.False(r.buffer.Success())
// reset wroteHeader
r.buffer.wroteHeader = false
r.buffer.WriteHeader(http.StatusOK)
r.True(r.buffer.Success())
}
func TestResponseBuffer(t *testing.T) {
suite.Run(t, &responseBufferTestSuite{})
}

138
src/pkg/artifact/dao/dao.go Normal file
View File

@ -0,0 +1,138 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"context"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/internal/orm"
"github.com/goharbor/harbor/src/pkg/q"
)
// DAO is the data access object interface for artifact
type DAO interface {
// Count returns the total count of artifacts according to the query
Count(ctx context.Context, query *q.Query) (total int64, err error)
// List artifacts according to the query
List(ctx context.Context, query *q.Query) (artifacts []*Artifact, err error)
// Get the artifact specified by ID
Get(ctx context.Context, id int64) (*Artifact, error)
// Create the artifact
Create(ctx context.Context, artifact *Artifact) (id int64, err error)
// Delete the artifact specified by ID
Delete(ctx context.Context, id int64) (err error)
// Update updates the artifact. Only the properties specified by "props" will be updated if it is set
Update(ctx context.Context, artifact *Artifact, props ...string) (err error)
// CreateReference creates the artifact reference
CreateReference(ctx context.Context, reference *ArtifactReference) (id int64, err error)
// ListReferences lists the artifact references according to the query
ListReferences(ctx context.Context, query *q.Query) (references []*ArtifactReference, err error)
// DeleteReferences deletes the references referenced by the artifact specified by parent ID
DeleteReferences(ctx context.Context, parentID int64) (err error)
}
// New returns an instance of the default DAO
func New() DAO {
return &dao{}
}
type dao struct{}
func (d *dao) Count(ctx context.Context, query *q.Query) (int64, error) {
if query != nil {
// ignore the page number and size
query = &q.Query{
Keywords: query.Keywords,
}
}
return orm.QuerySetter(ctx, &Artifact{}, query).Count()
}
func (d *dao) List(ctx context.Context, query *q.Query) ([]*Artifact, error) {
artifacts := []*Artifact{}
if _, err := orm.QuerySetter(ctx, &Artifact{}, query).All(&artifacts); err != nil {
return nil, err
}
return artifacts, nil
}
func (d *dao) Get(ctx context.Context, id int64) (*Artifact, error) {
artifact := &Artifact{
ID: id,
}
if err := orm.GetOrmer(ctx).Read(artifact); err != nil {
if e, ok := orm.IsNotFoundError(err, "artifact %d not found", id); ok {
err = e
}
return nil, err
}
return artifact, nil
}
func (d *dao) Create(ctx context.Context, artifact *Artifact) (int64, error) {
id, err := orm.GetOrmer(ctx).Insert(artifact)
if e, ok := orm.IsConflictError(err, "artifact %s already exists under the repository %d",
artifact.Digest, artifact.RepositoryID); ok {
err = e
}
return id, err
}
func (d *dao) Delete(ctx context.Context, id int64) error {
n, err := orm.GetOrmer(ctx).Delete(&Artifact{
ID: id,
})
if err != nil {
return err
}
if n == 0 {
return ierror.NotFoundError(nil).WithMessage("artifact %d not found", id)
}
return nil
}
func (d *dao) Update(ctx context.Context, artifact *Artifact, props ...string) error {
n, err := orm.GetOrmer(ctx).Update(artifact, props...)
if err != nil {
return err
}
if n == 0 {
return ierror.NotFoundError(nil).WithMessage("artifact %d not found", artifact.ID)
}
return nil
}
func (d *dao) CreateReference(ctx context.Context, reference *ArtifactReference) (int64, error) {
id, err := orm.GetOrmer(ctx).Insert(reference)
if e, ok := orm.IsConflictError(err, "reference already exists, parent artifact ID: %d, child artifact ID: %d",
reference.ParentID, reference.ChildID); ok {
err = e
}
return id, err
}
func (d *dao) ListReferences(ctx context.Context, query *q.Query) ([]*ArtifactReference, error) {
references := []*ArtifactReference{}
if _, err := orm.QuerySetter(ctx, &ArtifactReference{}, query).All(&references); err != nil {
return nil, err
}
return references, nil
}
func (d *dao) DeleteReferences(ctx context.Context, parentID int64) error {
// make sure the parent artifact exist
_, err := d.Get(ctx, parentID)
if err != nil {
return err
}
_, err = orm.QuerySetter(ctx, &ArtifactReference{}, &q.Query{
Keywords: map[string]interface{}{
"parent_id": parentID,
},
}).Delete()
return err
}

254
src/pkg/artifact/dao/dao_test.go Normal file
View File

@ -0,0 +1,254 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"errors"
common_dao "github.com/goharbor/harbor/src/common/dao"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/stretchr/testify/suite"
"testing"
"time"
)
var (
typee = "IMAGE"
mediaType = "application/vnd.oci.image.config.v1+json"
manifestMediaType = "application/vnd.oci.image.manifest.v1+json"
projectID int64 = 1
repositoryID int64 = 1
digest = "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180"
)
type daoTestSuite struct {
suite.Suite
dao DAO
artifactID int64
}
func (d *daoTestSuite) SetupSuite() {
d.dao = New()
common_dao.PrepareTestForPostgresSQL()
}
func (d *daoTestSuite) SetupTest() {
artifact := &Artifact{
Type: typee,
MediaType: mediaType,
ManifestMediaType: manifestMediaType,
ProjectID: projectID,
RepositoryID: repositoryID,
Digest: digest,
Size: 1024,
PushTime: time.Now(),
PullTime: time.Now(),
ExtraAttrs: `{"attr1":"value1"}`,
Annotations: `{"anno1":"value1"}`,
}
id, err := d.dao.Create(nil, artifact)
d.Require().Nil(err)
d.artifactID = id
}
func (d *daoTestSuite) TearDownTest() {
err := d.dao.Delete(nil, d.artifactID)
d.Require().Nil(err)
}
func (d *daoTestSuite) TestCount() {
// nil query
total, err := d.dao.Count(nil, nil)
d.Require().Nil(err)
d.True(total > 0)
// query by repository ID and digest
total, err = d.dao.Count(nil, &q.Query{
Keywords: map[string]interface{}{
"repository_id": repositoryID,
"digest": digest,
},
})
d.Require().Nil(err)
d.Equal(int64(1), total)
// query by repository ID and digest
total, err = d.dao.Count(nil, &q.Query{
Keywords: map[string]interface{}{
"repository_id": repositoryID,
"digest": digest,
},
})
d.Require().Nil(err)
d.Equal(int64(1), total)
// populate more data
id, err := d.dao.Create(nil, &Artifact{
Type: typee,
MediaType: mediaType,
ManifestMediaType: manifestMediaType,
ProjectID: projectID,
RepositoryID: repositoryID,
Digest: "sha256:digest",
})
d.Require().Nil(err)
defer func() {
err = d.dao.Delete(nil, id)
d.Require().Nil(err)
}()
// set pagination in query
total, err = d.dao.Count(nil, &q.Query{
PageNumber: 1,
PageSize: 1,
})
d.Require().Nil(err)
d.True(total > 1)
}
func (d *daoTestSuite) TestList() {
// nil query
artifacts, err := d.dao.List(nil, nil)
d.Require().Nil(err)
found := false
for _, artifact := range artifacts {
if artifact.ID == d.artifactID {
found = true
break
}
}
d.True(found)
// query by repository ID and digest
artifacts, err = d.dao.List(nil, &q.Query{
Keywords: map[string]interface{}{
"repository_id": repositoryID,
"digest": digest,
},
})
d.Require().Nil(err)
d.Require().Equal(1, len(artifacts))
d.Equal(d.artifactID, artifacts[0].ID)
}
func (d *daoTestSuite) TestGet() {
// get the non-exist artifact
_, err := d.dao.Get(nil, 10000)
d.Require().NotNil(err)
d.True(ierror.IsErr(err, ierror.NotFoundCode))
// get the exist artifact
artifact, err := d.dao.Get(nil, d.artifactID)
d.Require().Nil(err)
d.Require().NotNil(artifact)
d.Equal(d.artifactID, artifact.ID)
}
func (d *daoTestSuite) TestCreate() {
// the happy pass case is covered in Setup
// conflict
artifact := &Artifact{
Type: typee,
MediaType: mediaType,
ManifestMediaType: manifestMediaType,
ProjectID: projectID,
RepositoryID: repositoryID,
Digest: digest,
Size: 1024,
PushTime: time.Now(),
PullTime: time.Now(),
ExtraAttrs: `{"attr1":"value1"}`,
Annotations: `{"anno1":"value1"}`,
}
_, err := d.dao.Create(nil, artifact)
d.Require().NotNil(err)
d.True(ierror.IsErr(err, ierror.ConflictCode))
}
func (d *daoTestSuite) TestDelete() {
// the happy pass case is covered in TearDown
// not exist
err := d.dao.Delete(nil, 100021)
d.Require().NotNil(err)
var e *ierror.Error
d.Require().True(errors.As(err, &e))
d.Equal(ierror.NotFoundCode, e.Code)
}
func (d *daoTestSuite) TestUpdate() {
// pass
now := time.Now()
err := d.dao.Update(nil, &Artifact{
ID: d.artifactID,
PushTime: now,
}, "PushTime")
d.Require().Nil(err)
artifact, err := d.dao.Get(nil, d.artifactID)
d.Require().Nil(err)
d.Require().NotNil(artifact)
d.Equal(now.Unix(), artifact.PullTime.Unix())
// not exist
err = d.dao.Update(nil, &Artifact{
ID: 10000,
})
d.Require().NotNil(err)
var e *ierror.Error
d.Require().True(errors.As(err, &e))
d.Equal(ierror.NotFoundCode, e.Code)
}
func (d *daoTestSuite) TestReference() {
// create reference
id, err := d.dao.CreateReference(nil, &ArtifactReference{
ParentID: d.artifactID,
ChildID: 10000,
})
d.Require().Nil(err)
// conflict
_, err = d.dao.CreateReference(nil, &ArtifactReference{
ParentID: d.artifactID,
ChildID: 10000,
})
d.Require().NotNil(err)
d.True(ierror.IsErr(err, ierror.ConflictCode))
// list reference
references, err := d.dao.ListReferences(nil, &q.Query{
Keywords: map[string]interface{}{
"parent_id": d.artifactID,
},
})
d.Require().Equal(1, len(references))
d.Equal(id, references[0].ID)
// delete reference
err = d.dao.DeleteReferences(nil, d.artifactID)
d.Require().Nil(err)
// parent artifact not exist
err = d.dao.DeleteReferences(nil, 10000)
d.Require().NotNil(err)
var e *ierror.Error
d.Require().True(errors.As(err, &e))
d.Equal(ierror.NotFoundCode, e.Code)
}
func TestDaoTestSuite(t *testing.T) {
suite.Run(t, &daoTestSuite{})
}

113
src/pkg/artifact/manager.go
View File

@ -16,7 +16,11 @@ package artifact
import (
"context"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/artifact/dao"
"github.com/goharbor/harbor/src/pkg/q"
"strconv"
"strings"
"time"
)
@ -30,44 +34,121 @@ type Manager interface {
// List artifacts according to the query, returns all artifacts if query is nil
List(ctx context.Context, query *q.Query) (total int64, artifacts []*Artifact, err error)
// Get the artifact specified by the ID
Get(ctx context.Context, id int64) (*Artifact, error)
Get(ctx context.Context, id int64) (artifact *Artifact, err error)
// Create the artifact. If the artifact is an index, make sure all the artifacts it references
// already exist
Create(ctx context.Context, artifact *Artifact) (id int64, err error)
// Delete just deletes the artifact record. The underlying data of registry will be
// removed during garbage collection
Delete(ctx context.Context, id int64) error
Delete(ctx context.Context, id int64) (err error)
// UpdatePullTime updates the pull time of the artifact
UpdatePullTime(ctx context.Context, artifactID int64, time time.Time) error
UpdatePullTime(ctx context.Context, artifactID int64, time time.Time) (err error)
}
// NewManager returns an instance of the default manager
func NewManager() Manager {
return &manager{}
return &manager{
dao.New(),
}
}
var _ Manager = &manager{}
type manager struct {
dao dao.DAO
}
func (m *manager) List(ctx context.Context, query *q.Query) (total int64, artifacts []*Artifact, err error) {
// TODO implement
return 0, nil, nil
func (m *manager) List(ctx context.Context, query *q.Query) (int64, []*Artifact, error) {
total, err := m.dao.Count(ctx, query)
if err != nil {
return 0, nil, err
}
arts, err := m.dao.List(ctx, query)
if err != nil {
return 0, nil, err
}
var artifacts []*Artifact
for _, art := range arts {
artifact, err := m.assemble(ctx, art)
if err != nil {
return 0, nil, err
}
artifacts = append(artifacts, artifact)
}
return total, artifacts, nil
}
func (m *manager) Get(ctx context.Context, id int64) (*Artifact, error) {
// TODO implement
return nil, nil
art, err := m.dao.Get(ctx, id)
if err != nil {
return nil, err
}
return m.assemble(ctx, art)
}
func (m *manager) Create(ctx context.Context, artifact *Artifact) (id int64, err error) {
// TODO implement
return 0, nil
func (m *manager) Create(ctx context.Context, artifact *Artifact) (int64, error) {
id, err := m.dao.Create(ctx, artifact.To())
if err != nil {
return 0, err
}
for _, reference := range artifact.References {
reference.ParentID = id
if _, err = m.dao.CreateReference(ctx, reference.To()); err != nil {
return 0, err
}
}
return id, nil
}
func (m *manager) Delete(ctx context.Context, id int64) error {
// TODO implement
return nil
// check whether the artifact is referenced by other artifacts
references, err := m.dao.ListReferences(ctx, &q.Query{
Keywords: map[string]interface{}{
"child_id": id,
},
})
if err != nil {
return err
}
if len(references) > 0 {
var ids []string
for _, reference := range references {
ids = append(ids, strconv.FormatInt(reference.ParentID, 10))
}
return ierror.PreconditionFailedError(nil).
WithMessage("artifact %d is referenced by other artifacts: %s", id, strings.Join(ids, ","))
}
// delete references
if err := m.dao.DeleteReferences(ctx, id); err != nil {
return err
}
// delete artifact
return m.dao.Delete(ctx, id)
}
func (m *manager) UpdatePullTime(ctx context.Context, artifactID int64, time time.Time) error {
// TODO implement
return nil
return m.dao.Update(ctx, &dao.Artifact{
ID: artifactID,
PullTime: time,
}, "PullTime")
}
// assemble the artifact with references populated
func (m *manager) assemble(ctx context.Context, art *dao.Artifact) (*Artifact, error) {
artifact := &Artifact{}
// convert from database object
artifact.From(art)
// populate the references
refs, err := m.dao.ListReferences(ctx, &q.Query{
Keywords: map[string]interface{}{
"parent_id": artifact.ID,
},
})
if err != nil {
return nil, err
}
for _, ref := range refs {
reference := &Reference{}
reference.From(ref)
artifact.References = append(artifact.References, reference)
}
return artifact, nil
}

203
src/pkg/artifact/manager_test.go
View File

@ -14,4 +14,205 @@
package artifact
// TODO add test cases after implementing the manager
import (
"context"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/artifact/dao"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"testing"
"time"
)
type fakeDao struct {
mock.Mock
}
func (f *fakeDao) Count(ctx context.Context, query *q.Query) (int64, error) {
args := f.Called()
return int64(args.Int(0)), args.Error(1)
}
func (f *fakeDao) List(ctx context.Context, query *q.Query) ([]*dao.Artifact, error) {
args := f.Called()
return args.Get(0).([]*dao.Artifact), args.Error(1)
}
func (f *fakeDao) Get(ctx context.Context, id int64) (*dao.Artifact, error) {
args := f.Called()
return args.Get(0).(*dao.Artifact), args.Error(1)
}
func (f *fakeDao) Create(ctx context.Context, artifact *dao.Artifact) (int64, error) {
args := f.Called()
return int64(args.Int(0)), args.Error(1)
}
func (f *fakeDao) Delete(ctx context.Context, id int64) error {
args := f.Called()
return args.Error(0)
}
func (f *fakeDao) Update(ctx context.Context, artifact *dao.Artifact, props ...string) error {
args := f.Called()
return args.Error(0)
}
func (f *fakeDao) CreateReference(ctx context.Context, reference *dao.ArtifactReference) (int64, error) {
args := f.Called()
return int64(args.Int(0)), args.Error(1)
}
func (f *fakeDao) ListReferences(ctx context.Context, query *q.Query) ([]*dao.ArtifactReference, error) {
args := f.Called()
return args.Get(0).([]*dao.ArtifactReference), args.Error(1)
}
func (f *fakeDao) DeleteReferences(ctx context.Context, parentID int64) error {
args := f.Called()
return args.Error(0)
}
type managerTestSuite struct {
suite.Suite
mgr *manager
dao *fakeDao
}
func (m *managerTestSuite) SetupTest() {
m.dao = &fakeDao{}
m.mgr = &manager{
dao: m.dao,
}
}
func (m *managerTestSuite) TestAssemble() {
art := &dao.Artifact{
ID: 1,
Type: "IMAGE",
MediaType: "application/vnd.oci.image.config.v1+json",
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
ProjectID: 1,
RepositoryID: 1,
Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
Size: 1024,
PushTime: time.Now(),
PullTime: time.Now(),
ExtraAttrs: `{"attr1":"value1"}`,
Annotations: `{"anno1":"value1"}`,
}
m.dao.On("ListReferences").Return([]*dao.ArtifactReference{
{
ID: 1,
ParentID: 1,
ChildID: 2,
},
{
ID: 2,
ParentID: 1,
ChildID: 3,
},
}, nil)
artifact, err := m.mgr.assemble(nil, art)
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
m.Require().NotNil(artifact)
m.Equal(art.ID, artifact.ID)
m.Equal(2, len(artifact.References))
}
func (m *managerTestSuite) TestList() {
art := &dao.Artifact{
ID: 1,
Type: "IMAGE",
MediaType: "application/vnd.oci.image.config.v1+json",
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
ProjectID: 1,
RepositoryID: 1,
Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
Size: 1024,
PushTime: time.Now(),
PullTime: time.Now(),
ExtraAttrs: `{"attr1":"value1"}`,
Annotations: `{"anno1":"value1"}`,
}
m.dao.On("Count", mock.Anything).Return(1, nil)
m.dao.On("List", mock.Anything).Return([]*dao.Artifact{art}, nil)
m.dao.On("ListReferences").Return([]*dao.ArtifactReference{}, nil)
total, artifacts, err := m.mgr.List(nil, nil)
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
m.Equal(int64(1), total)
m.Equal(1, len(artifacts))
m.Equal(art.ID, artifacts[0].ID)
}
func (m *managerTestSuite) TestGet() {
art := &dao.Artifact{
ID: 1,
Type: "IMAGE",
MediaType: "application/vnd.oci.image.config.v1+json",
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
ProjectID: 1,
RepositoryID: 1,
Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
Size: 1024,
PushTime: time.Now(),
PullTime: time.Now(),
ExtraAttrs: `{"attr1":"value1"}`,
Annotations: `{"anno1":"value1"}`,
}
m.dao.On("Get", mock.Anything).Return(art, nil)
m.dao.On("ListReferences").Return([]*dao.ArtifactReference{}, nil)
artifact, err := m.mgr.Get(nil, 1)
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
m.Require().NotNil(artifact)
m.Equal(art.ID, artifact.ID)
}
func (m *managerTestSuite) TestCreate() {
m.dao.On("Create", mock.Anything).Return(1, nil)
m.dao.On("CreateReference").Return(1, nil)
id, err := m.mgr.Create(nil, &Artifact{
References: []*Reference{
{
ChildID: 2,
},
},
})
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
m.Equal(int64(1), id)
}
func (m *managerTestSuite) TestDelete() {
// referenced by other artifacts, delete failed
m.dao.On("ListReferences").Return([]*dao.ArtifactReference{
{
ParentID: 1,
ChildID: 1,
},
}, nil)
err := m.mgr.Delete(nil, 1)
m.Require().NotNil(err)
m.dao.AssertExpectations(m.T())
e, ok := err.(*ierror.Error)
m.Require().True(ok)
m.Equal(ierror.PreconditionCode, e.Code)
// reset the mock
m.SetupTest()
// // referenced by no artifacts
m.dao.On("ListReferences").Return([]*dao.ArtifactReference{}, nil)
m.dao.On("Delete", mock.Anything).Return(nil)
m.dao.On("DeleteReferences").Return(nil)
err = m.mgr.Delete(nil, 1)
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
}
func (m *managerTestSuite) TestUpdatePullTime() {
m.dao.On("Update", mock.Anything).Return(nil)
err := m.mgr.UpdatePullTime(nil, 1, time.Now())
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
}
func TestManager(t *testing.T) {
suite.Run(t, &managerTestSuite{})
}

5
src/pkg/project/manager.go
View File

@ -21,6 +21,11 @@ import (
"github.com/goharbor/harbor/src/common/models"
)
var (
// Mgr is the global project manager
Mgr = New()
)
// Manager is used for project management
// currently, the interface only defines the methods needed for tag retention
// will expand it when doing refactor

108
src/pkg/repository/dao/dao.go Normal file
View File

@ -0,0 +1,108 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"context"
"github.com/goharbor/harbor/src/common/models"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/internal/orm"
"github.com/goharbor/harbor/src/pkg/q"
)
// DAO is the data access object interface for repository
type DAO interface {
// Count returns the total count of repositories according to the query
Count(ctx context.Context, query *q.Query) (count int64, err error)
// List repositories according to the query
List(ctx context.Context, query *q.Query) (repositories []*models.RepoRecord, err error)
// Get the repository specified by ID
Get(ctx context.Context, id int64) (repository *models.RepoRecord, err error)
// Create the repository
Create(ctx context.Context, repository *models.RepoRecord) (id int64, err error)
// Delete the repository specified by ID
Delete(ctx context.Context, id int64) (err error)
// Update updates the repository. Only the properties specified by "props" will be updated if it is set
Update(ctx context.Context, repository *models.RepoRecord, props ...string) (err error)
}
// New returns an instance of the default DAO
func New() DAO {
return &dao{}
}
type dao struct{}
func (d *dao) Count(ctx context.Context, query *q.Query) (int64, error) {
if query != nil {
// ignore the page number and size
query = &q.Query{
Keywords: query.Keywords,
}
}
return orm.QuerySetter(ctx, &models.RepoRecord{}, query).Count()
}
func (d *dao) List(ctx context.Context, query *q.Query) ([]*models.RepoRecord, error) {
repositories := []*models.RepoRecord{}
if _, err := orm.QuerySetter(ctx, &models.RepoRecord{}, query).All(&repositories); err != nil {
return nil, err
}
return repositories, nil
}
func (d *dao) Get(ctx context.Context, id int64) (*models.RepoRecord, error) {
repository := &models.RepoRecord{
RepositoryID: id,
}
if err := orm.GetOrmer(ctx).Read(repository); err != nil {
if e, ok := orm.IsNotFoundError(err, "repository %d not found", id); ok {
err = e
}
return nil, err
}
return repository, nil
}
func (d *dao) Create(ctx context.Context, repository *models.RepoRecord) (int64, error) {
id, err := orm.GetOrmer(ctx).Insert(repository)
if e, ok := orm.IsConflictError(err, "repository %s already exists", repository.Name); ok {
err = e
}
return id, err
}
func (d *dao) Delete(ctx context.Context, id int64) error {
n, err := orm.GetOrmer(ctx).Delete(&models.RepoRecord{
RepositoryID: id,
})
if err != nil {
return err
}
if n == 0 {
return ierror.NotFoundError(nil).WithMessage("repository %d not found", id)
}
return nil
}
func (d *dao) Update(ctx context.Context, repository *models.RepoRecord, props ...string) error {
n, err := orm.GetOrmer(ctx).Update(repository, props...)
if err != nil {
return err
}
if n == 0 {
return ierror.NotFoundError(nil).WithMessage("repository %d not found", repository.RepositoryID)
}
return nil
}

162
src/pkg/repository/dao/dao_test.go Normal file
View File

@ -0,0 +1,162 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"errors"
"fmt"
common_dao "github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/stretchr/testify/suite"
"testing"
"time"
)
var (
repository = fmt.Sprintf("library/%d", time.Now().Unix())
)
type daoTestSuite struct {
suite.Suite
dao DAO
id int64
}
func (d *daoTestSuite) SetupSuite() {
d.dao = New()
common_dao.PrepareTestForPostgresSQL()
}
func (d *daoTestSuite) SetupTest() {
repository := &models.RepoRecord{
Name: repository,
ProjectID: 1,
Description: "",
}
id, err := d.dao.Create(nil, repository)
d.Require().Nil(err)
d.id = id
}
func (d *daoTestSuite) TearDownTest() {
err := d.dao.Delete(nil, d.id)
d.Require().Nil(err)
}
func (d *daoTestSuite) TestCount() {
// nil query
total, err := d.dao.Count(nil, nil)
d.Require().Nil(err)
d.True(total > 0)
// query by name
total, err = d.dao.Count(nil, &q.Query{
Keywords: map[string]interface{}{
"name": repository,
},
})
d.Require().Nil(err)
d.Equal(int64(1), total)
}
func (d *daoTestSuite) TestList() {
// nil query
repositories, err := d.dao.List(nil, nil)
d.Require().Nil(err)
found := false
for _, repository := range repositories {
if repository.RepositoryID == d.id {
found = true
break
}
}
d.True(found)
// query by name
repositories, err = d.dao.List(nil, &q.Query{
Keywords: map[string]interface{}{
"name": repository,
},
})
d.Require().Nil(err)
d.Require().Equal(1, len(repositories))
d.Equal(d.id, repositories[0].RepositoryID)
}
func (d *daoTestSuite) TestGet() {
// get the non-exist repository
_, err := d.dao.Get(nil, 10000)
d.Require().NotNil(err)
d.True(ierror.IsErr(err, ierror.NotFoundCode))
// get the exist repository
repository, err := d.dao.Get(nil, d.id)
d.Require().Nil(err)
d.Require().NotNil(repository)
d.Equal(d.id, repository.RepositoryID)
}
func (d *daoTestSuite) TestCreate() {
// the happy pass case is covered in Setup
// conflict
repository := &models.RepoRecord{
Name: repository,
ProjectID: 1,
}
_, err := d.dao.Create(nil, repository)
d.Require().NotNil(err)
d.True(ierror.IsErr(err, ierror.ConflictCode))
}
func (d *daoTestSuite) TestDelete() {
// the happy pass case is covered in TearDown
// not exist
err := d.dao.Delete(nil, 100021)
d.Require().NotNil(err)
var e *ierror.Error
d.Require().True(errors.As(err, &e))
d.Equal(ierror.NotFoundCode, e.Code)
}
func (d *daoTestSuite) TestUpdate() {
// pass
err := d.dao.Update(nil, &models.RepoRecord{
RepositoryID: d.id,
PullCount: 1,
}, "PullCount")
d.Require().Nil(err)
repository, err := d.dao.Get(nil, d.id)
d.Require().Nil(err)
d.Require().NotNil(repository)
d.Equal(int64(1), repository.PullCount)
// not exist
err = d.dao.Update(nil, &models.RepoRecord{
RepositoryID: 10000,
})
d.Require().NotNil(err)
var e *ierror.Error
d.Require().True(errors.As(err, &e))
d.Equal(ierror.NotFoundCode, e.Code)
}
func TestDaoTestSuite(t *testing.T) {
suite.Run(t, &daoTestSuite{})
}

68
src/pkg/repository/manager.go
View File

@ -15,47 +15,63 @@
package repository
import (
"github.com/goharbor/harbor/src/chartserver"
"github.com/goharbor/harbor/src/common/dao"
"context"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/project"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/repository/dao"
)
// Mgr is the global repository manager instance
var Mgr = New()
// Manager is used for repository management
// currently, the interface only defines the methods needed for tag retention
// will expand it when doing refactor
type Manager interface {
// List image repositories under the project specified by the ID
ListImageRepositories(projectID int64) ([]*models.RepoRecord, error)
// List chart repositories under the project specified by the ID
ListChartRepositories(projectID int64) ([]*chartserver.ChartInfo, error)
// List repositories according to the query
List(ctx context.Context, query *q.Query) (total int64, repositories []*models.RepoRecord, err error)
// Get the repository specified by ID
Get(ctx context.Context, id int64) (repository *models.RepoRecord, err error)
// Create a repository
Create(ctx context.Context, repository *models.RepoRecord) (id int64, err error)
// Delete the repository specified by ID
Delete(ctx context.Context, id int64) (err error)
// Update updates the repository. Only the properties specified by "props" will be updated if it is set
Update(ctx context.Context, repository *models.RepoRecord, props ...string) (err error)
}
// New returns a default implementation of Manager
func New(projectMgr project.Manager, chartCtl *chartserver.Controller) Manager {
func New() Manager {
return &manager{
projectMgr: projectMgr,
chartCtl: chartCtl,
dao: dao.New(),
}
}
type manager struct {
projectMgr project.Manager
chartCtl *chartserver.Controller
dao dao.DAO
}
// List image repositories under the project specified by the ID
func (m *manager) ListImageRepositories(projectID int64) ([]*models.RepoRecord, error) {
return dao.GetRepositories(&models.RepositoryQuery{
ProjectIDs: []int64{projectID},
})
}
// List chart repositories under the project specified by the ID
func (m *manager) ListChartRepositories(projectID int64) ([]*chartserver.ChartInfo, error) {
project, err := m.projectMgr.Get(projectID)
func (m *manager) List(ctx context.Context, query *q.Query) (int64, []*models.RepoRecord, error) {
total, err := m.dao.Count(ctx, query)
if err != nil {
return nil, err
return 0, nil, err
}
return m.chartCtl.ListCharts(project.Name)
repositories, err := m.dao.List(ctx, query)
if err != nil {
return 0, nil, err
}
return total, repositories, nil
}
func (m *manager) Get(ctx context.Context, id int64) (*models.RepoRecord, error) {
return m.dao.Get(ctx, id)
}
func (m *manager) Create(ctx context.Context, repository *models.RepoRecord) (int64, error) {
return m.dao.Create(ctx, repository)
}
func (m *manager) Delete(ctx context.Context, id int64) error {
return m.dao.Delete(ctx, id)
}
func (m *manager) Update(ctx context.Context, repository *models.RepoRecord, props ...string) error {
return m.dao.Update(ctx, repository)
}

127
src/pkg/repository/manager_test.go Normal file
View File

@ -0,0 +1,127 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package repository
import (
"context"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"testing"
)
type fakeDao struct {
mock.Mock
}
func (f *fakeDao) Count(ctx context.Context, query *q.Query) (int64, error) {
args := f.Called()
return int64(args.Int(0)), args.Error(1)
}
func (f *fakeDao) List(ctx context.Context, query *q.Query) ([]*models.RepoRecord, error) {
args := f.Called()
return args.Get(0).([]*models.RepoRecord), args.Error(1)
}
func (f *fakeDao) Get(ctx context.Context, id int64) (*models.RepoRecord, error) {
args := f.Called()
return args.Get(0).(*models.RepoRecord), args.Error(1)
}
func (f *fakeDao) Create(ctx context.Context, repository *models.RepoRecord) (int64, error) {
args := f.Called()
return int64(args.Int(0)), args.Error(1)
}
func (f *fakeDao) Delete(ctx context.Context, id int64) error {
args := f.Called()
return args.Error(0)
}
func (f *fakeDao) Update(ctx context.Context, repository *models.RepoRecord, props ...string) error {
args := f.Called()
return args.Error(0)
}
type managerTestSuite struct {
suite.Suite
mgr *manager
dao *fakeDao
}
func (m *managerTestSuite) SetupTest() {
m.dao = &fakeDao{}
m.mgr = &manager{
dao: m.dao,
}
}
func (m *managerTestSuite) TestList() {
repository := &models.RepoRecord{
RepositoryID: 1,
ProjectID: 1,
Name: "library/hello-world",
}
m.dao.On("Count", mock.Anything).Return(1, nil)
m.dao.On("List", mock.Anything).Return([]*models.RepoRecord{repository}, nil)
total, repositories, err := m.mgr.List(nil, nil)
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
m.Equal(int64(1), total)
m.Equal(1, len(repositories))
m.Equal(repository.RepositoryID, repositories[0].RepositoryID)
}
func (m *managerTestSuite) TestGet() {
repository := &models.RepoRecord{
RepositoryID: 1,
ProjectID: 1,
Name: "library/hello-world",
}
m.dao.On("Get", mock.Anything).Return(repository, nil)
repo, err := m.mgr.Get(nil, 1)
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
m.Require().NotNil(repo)
m.Equal(repository.RepositoryID, repo.RepositoryID)
}
func (m *managerTestSuite) TestCreate() {
m.dao.On("Create", mock.Anything).Return(1, nil)
id, err := m.mgr.Create(nil, &models.RepoRecord{
ProjectID: 1,
Name: "library/hello-world",
})
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
m.Equal(int64(1), id)
}
func (m *managerTestSuite) TestDelete() {
m.dao.On("Delete", mock.Anything).Return(nil)
err := m.mgr.Delete(nil, 1)
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
}
func (m *managerTestSuite) TestUpdate() {
m.dao.On("Update", mock.Anything).Return(nil)
err := m.mgr.Update(nil, &models.RepoRecord{
RepositoryID: 1,
})
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
}
func TestManager(t *testing.T) {
suite.Run(t, &managerTestSuite{})
}

5
src/pkg/retention/controller_test.go
View File

@ -1,6 +1,7 @@
package retention
import (
htesting "github.com/goharbor/harbor/src/testing"
"strings"
"testing"
@ -28,7 +29,7 @@ func TestController(t *testing.T) {
func (s *ControllerTestSuite) TestPolicy() {
projectMgr := &fakeProjectManager{}
repositoryMgr := &fakeRepositoryManager{}
repositoryMgr := &htesting.FakeRepositoryManager{}
retentionScheduler := &fakeRetentionScheduler{}
retentionLauncher := &fakeLauncher{}
retentionMgr := NewManager()
@ -126,7 +127,7 @@ func (s *ControllerTestSuite) TestPolicy() {
func (s *ControllerTestSuite) TestExecution() {
projectMgr := &fakeProjectManager{}
repositoryMgr := &fakeRepositoryManager{}
repositoryMgr := &htesting.FakeRepositoryManager{}
retentionScheduler := &fakeRetentionScheduler{}
retentionLauncher := &fakeLauncher{}
retentionMgr := NewManager()

8
src/pkg/retention/launcher.go
View File

@ -29,6 +29,7 @@ import (
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/pkg/art"
"github.com/goharbor/harbor/src/pkg/project"
pq "github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/repository"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/retention/policy/lwp"
@ -346,7 +347,12 @@ func getRepositories(projectMgr project.Manager, repositoryMgr repository.Manage
}
*/
// get image repositories
imageRepositories, err := repositoryMgr.ListImageRepositories(projectID)
// TODO set the context which contains the ORM
_, imageRepositories, err := repositoryMgr.List(nil, &pq.Query{
Keywords: map[string]interface{}{
"ProjectID": projectID,
},
})
if err != nil {
return nil, err
}

53
src/pkg/retention/launcher_test.go
View File

@ -16,14 +16,13 @@ package retention
import (
"fmt"
htesting "github.com/goharbor/harbor/src/testing"
"testing"
"github.com/goharbor/harbor/src/chartserver"
"github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/models"
_ "github.com/goharbor/harbor/src/pkg/art/selectors/doublestar"
"github.com/goharbor/harbor/src/pkg/project"
"github.com/goharbor/harbor/src/pkg/repository"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
"github.com/goharbor/harbor/src/pkg/retention/q"
@ -62,18 +61,6 @@ func (f *fakeProjectManager) Get(idOrName interface{}) (*models.Project, error)
return nil, fmt.Errorf("invalid parameter: %v, should be ID(int64) or name(string)", idOrName)
}
type fakeRepositoryManager struct {
imageRepositories []*models.RepoRecord
chartRepositories []*chartserver.ChartInfo
}
func (f *fakeRepositoryManager) ListImageRepositories(projectID int64) ([]*models.RepoRecord, error) {
return f.imageRepositories, nil
}
func (f *fakeRepositoryManager) ListChartRepositories(projectID int64) ([]*chartserver.ChartInfo, error) {
return f.chartRepositories, nil
}
type fakeRetentionManager struct{}
func (f *fakeRetentionManager) GetTotalOfRetentionExecs(policyID int64) (int64, error) {
@ -145,7 +132,7 @@ func (f *fakeRetentionManager) ListHistories(executionID int64, query *q.Query)
type launchTestSuite struct {
suite.Suite
projectMgr project.Manager
repositoryMgr repository.Manager
repositoryMgr *htesting.FakeRepositoryManager
retentionMgr Manager
jobserviceClient job.Client
}
@ -163,21 +150,7 @@ func (l *launchTestSuite) SetupTest() {
projects: []*models.Project{
pro1, pro2,
}}
l.repositoryMgr = &fakeRepositoryManager{
imageRepositories: []*models.RepoRecord{
{
Name: "library/image",
},
{
Name: "test/image",
},
},
chartRepositories: []*chartserver.ChartInfo{
{
Name: "chart",
},
},
}
l.repositoryMgr = &htesting.FakeRepositoryManager{}
l.retentionMgr = &fakeRetentionManager{}
l.jobserviceClient = &hjob.MockJobClient{
JobUUID: []string{"1"},
@ -193,9 +166,17 @@ func (l *launchTestSuite) TestGetProjects() {
}
func (l *launchTestSuite) TestGetRepositories() {
l.repositoryMgr.On("List").Return(1, []*models.RepoRecord{
{
RepositoryID: 1,
ProjectID: 1,
Name: "library/image",
},
}, nil)
repositories, err := getRepositories(l.projectMgr, l.repositoryMgr, 1, true)
require.Nil(l.T(), err)
assert.Equal(l.T(), 2, len(repositories))
l.repositoryMgr.AssertExpectations(l.T())
assert.Equal(l.T(), 1, len(repositories))
assert.Equal(l.T(), "library", repositories[0].Namespace)
assert.Equal(l.T(), "image", repositories[0].Repository)
assert.Equal(l.T(), "image", repositories[0].Kind)
@ -231,6 +212,13 @@ func (l *launchTestSuite) TestLaunch() {
require.NotNil(l.T(), err)
// system scope
l.repositoryMgr.On("List").Return(2, []*models.RepoRecord{
{
RepositoryID: 1,
ProjectID: 1,
Name: "library/image",
},
}, nil)
ply = &policy.Metadata{
Scope: &policy.Scope{
Level: "system",
@ -277,7 +265,8 @@ func (l *launchTestSuite) TestLaunch() {
}
n, err = launcher.Launch(ply, 1, false)
require.Nil(l.T(), err)
assert.Equal(l.T(), int64(2), n)
l.repositoryMgr.AssertExpectations(l.T())
assert.Equal(l.T(), int64(1), n)
}
func (l *launchTestSuite) TestStop() {

90
src/pkg/tag/dao/dao.go
View File

@ -15,10 +15,96 @@
package dao
import (
"github.com/astaxie/beego/orm"
"context"
beego_orm "github.com/astaxie/beego/orm"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/internal/orm"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/tag/model/tag"
)
func init() {
orm.RegisterModel(&tag.Tag{})
beego_orm.RegisterModel(&tag.Tag{})
}
// DAO is the data access object for tag
type DAO interface {
// Count returns the total count of tags according to the query
Count(ctx context.Context, query *q.Query) (total int64, err error)
// List tags according to the query
List(ctx context.Context, query *q.Query) (tags []*tag.Tag, err error)
// Get the tag specified by ID
Get(ctx context.Context, id int64) (tag *tag.Tag, err error)
// Create the tag
Create(ctx context.Context, tag *tag.Tag) (id int64, err error)
// Update the tag. Only the properties specified by "props" will be updated if it is set
Update(ctx context.Context, tag *tag.Tag, props ...string) (err error)
// Delete the tag specified by ID
Delete(ctx context.Context, id int64) (err error)
}
// New returns an instance of the default DAO
func New() DAO {
return &dao{}
}
type dao struct{}
func (d *dao) Count(ctx context.Context, query *q.Query) (int64, error) {
if query != nil {
// ignore the page number and size
query = &q.Query{
Keywords: query.Keywords,
}
}
return orm.QuerySetter(ctx, &tag.Tag{}, query).Count()
}
func (d *dao) List(ctx context.Context, query *q.Query) ([]*tag.Tag, error) {
tags := []*tag.Tag{}
if _, err := orm.QuerySetter(ctx, &tag.Tag{}, query).All(&tags); err != nil {
return nil, err
}
return tags, nil
}
func (d *dao) Get(ctx context.Context, id int64) (*tag.Tag, error) {
tag := &tag.Tag{
ID: id,
}
if err := orm.GetOrmer(ctx).Read(tag); err != nil {
if e, ok := orm.IsNotFoundError(err, "tag %d not found", id); ok {
err = e
}
return nil, err
}
return tag, nil
}
func (d *dao) Create(ctx context.Context, tag *tag.Tag) (int64, error) {
id, err := orm.GetOrmer(ctx).Insert(tag)
if e, ok := orm.IsConflictError(err, "tag %s already exists under the repository %d",
tag.Name, tag.RepositoryID); ok {
err = e
}
return id, err
}
func (d *dao) Update(ctx context.Context, tag *tag.Tag, props ...string) error {
n, err := orm.GetOrmer(ctx).Update(tag, props...)
if err != nil {
return err
}
if n == 0 {
return ierror.NotFoundError(nil).WithMessage("tag %d not found", tag.ID)
}
return nil
}
func (d *dao) Delete(ctx context.Context, id int64) error {
n, err := orm.GetOrmer(ctx).Delete(&tag.Tag{
ID: id,
})
if err != nil {
return err
}
if n == 0 {
return ierror.NotFoundError(nil).WithMessage("tag %d not found", id)
}
return nil
}

169
src/pkg/tag/dao/dao_test.go Normal file
View File

@ -0,0 +1,169 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"errors"
common_dao "github.com/goharbor/harbor/src/common/dao"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/tag/model/tag"
"github.com/stretchr/testify/suite"
"testing"
"time"
)
var (
repositoryID int64 = 1000
artifactID int64 = 1000
name = "latest"
)
type daoTestSuite struct {
suite.Suite
dao DAO
tagID int64
}
func (d *daoTestSuite) SetupSuite() {
d.dao = New()
common_dao.PrepareTestForPostgresSQL()
}
func (d *daoTestSuite) SetupTest() {
tag := &tag.Tag{
RepositoryID: repositoryID,
ArtifactID: artifactID,
Name: name,
PushTime: time.Time{},
PullTime: time.Time{},
}
id, err := d.dao.Create(nil, tag)
d.Require().Nil(err)
d.tagID = id
}
func (d *daoTestSuite) TearDownTest() {
err := d.dao.Delete(nil, d.tagID)
d.Require().Nil(err)
}
func (d *daoTestSuite) TestCount() {
// nil query
total, err := d.dao.Count(nil, nil)
d.Require().Nil(err)
d.True(total > 0)
// query by repository ID and name
total, err = d.dao.Count(nil, &q.Query{
Keywords: map[string]interface{}{
"repository_id": repositoryID,
"name": name,
},
})
d.Require().Nil(err)
d.Equal(int64(1), total)
}
func (d *daoTestSuite) TestList() {
// nil query
tags, err := d.dao.List(nil, nil)
d.Require().Nil(err)
found := false
for _, tag := range tags {
if tag.ID == d.tagID {
found = true
break
}
}
d.True(found)
// query by repository ID and name
tags, err = d.dao.List(nil, &q.Query{
Keywords: map[string]interface{}{
"repository_id": repositoryID,
"name": name,
},
})
d.Require().Nil(err)
d.Require().Equal(1, len(tags))
d.Equal(d.tagID, tags[0].ID)
}
func (d *daoTestSuite) TestGet() {
// get the non-exist tag
_, err := d.dao.Get(nil, 10000)
d.Require().NotNil(err)
d.True(ierror.IsErr(err, ierror.NotFoundCode))
// get the exist tag
tag, err := d.dao.Get(nil, d.tagID)
d.Require().Nil(err)
d.Require().NotNil(tag)
d.Equal(d.tagID, tag.ID)
}
func (d *daoTestSuite) TestCreate() {
// the happy pass case is covered in Setup
// conflict
tag := &tag.Tag{
RepositoryID: repositoryID,
ArtifactID: artifactID,
Name: name,
PushTime: time.Time{},
PullTime: time.Time{},
}
_, err := d.dao.Create(nil, tag)
d.Require().NotNil(err)
d.True(ierror.IsErr(err, ierror.ConflictCode))
}
func (d *daoTestSuite) TestDelete() {
// happy pass is covered in TearDown
// not exist
err := d.dao.Delete(nil, 10000)
d.Require().NotNil(err)
var e *ierror.Error
d.Require().True(errors.As(err, &e))
d.Equal(ierror.NotFoundCode, e.Code)
}
func (d *daoTestSuite) TestUpdate() {
// pass
err := d.dao.Update(nil, &tag.Tag{
ID: d.tagID,
ArtifactID: 2,
}, "ArtifactID")
d.Require().Nil(err)
tg, err := d.dao.Get(nil, d.tagID)
d.Require().Nil(err)
d.Require().NotNil(tg)
d.Equal(int64(2), tg.ArtifactID)
// not exist
err = d.dao.Update(nil, &tag.Tag{
ID: 10000,
})
d.Require().NotNil(err)
var e *ierror.Error
d.Require().True(errors.As(err, &e))
d.Equal(ierror.NotFoundCode, e.Code)
}
func TestDaoTestSuite(t *testing.T) {
suite.Run(t, &daoTestSuite{})
}

42
src/pkg/tag/manager.go
View File

@ -17,6 +17,7 @@ package tag
import (
"context"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/tag/dao"
"github.com/goharbor/harbor/src/pkg/tag/model/tag"
)
@ -33,14 +34,47 @@ type Manager interface {
Get(ctx context.Context, id int64) (tag *tag.Tag, err error)
// Create the tag and returns the ID
Create(ctx context.Context, tag *tag.Tag) (id int64, err error)
// Update the tag
Update(ctx context.Context, tag *tag.Tag) (err error)
// Update the tag. Only the properties specified by "props" will be updated if it is set
Update(ctx context.Context, tag *tag.Tag, props ...string) (err error)
// Delete the tag specified by ID
Delete(ctx context.Context, id int64) (err error)
}
// NewManager creates an instance of the default tag manager
func NewManager() Manager {
// TODO implement
return nil
return &manager{
dao: dao.New(),
}
}
type manager struct {
dao dao.DAO
}
func (m *manager) List(ctx context.Context, query *q.Query) (int64, []*tag.Tag, error) {
total, err := m.dao.Count(ctx, query)
if err != nil {
return 0, nil, err
}
tags, err := m.dao.List(ctx, query)
if err != nil {
return 0, nil, err
}
return total, tags, nil
}
func (m *manager) Get(ctx context.Context, id int64) (*tag.Tag, error) {
return m.dao.Get(ctx, id)
}
func (m *manager) Create(ctx context.Context, tag *tag.Tag) (int64, error) {
return m.dao.Create(ctx, tag)
}
func (m *manager) Update(ctx context.Context, tag *tag.Tag, props ...string) error {
return m.dao.Update(ctx, tag, props...)
}
func (m *manager) Delete(ctx context.Context, id int64) error {
return m.dao.Delete(ctx, id)
}

118
src/pkg/tag/manager_test.go Normal file
View File

@ -0,0 +1,118 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tag
import (
"context"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/tag/model/tag"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"testing"
"time"
)
type fakeDao struct {
mock.Mock
}
func (f *fakeDao) Count(ctx context.Context, query *q.Query) (int64, error) {
args := f.Called()
return int64(args.Int(0)), args.Error(1)
}
func (f *fakeDao) List(ctx context.Context, query *q.Query) ([]*tag.Tag, error) {
args := f.Called()
return args.Get(0).([]*tag.Tag), args.Error(1)
}
func (f *fakeDao) Get(ctx context.Context, id int64) (*tag.Tag, error) {
args := f.Called()
return args.Get(0).(*tag.Tag), args.Error(1)
}
func (f *fakeDao) Create(ctx context.Context, tag *tag.Tag) (int64, error) {
args := f.Called()
return int64(args.Int(0)), args.Error(1)
}
func (f *fakeDao) Update(ctx context.Context, tag *tag.Tag, props ...string) error {
args := f.Called()
return args.Error(0)
}
func (f *fakeDao) Delete(ctx context.Context, id int64) error {
args := f.Called()
return args.Error(0)
}
type managerTestSuite struct {
suite.Suite
mgr *manager
dao *fakeDao
}
func (m *managerTestSuite) SetupTest() {
m.dao = &fakeDao{}
m.mgr = &manager{
dao: m.dao,
}
}
func (m *managerTestSuite) TestList() {
tg := &tag.Tag{
ID: 1,
RepositoryID: 1,
ArtifactID: 1,
Name: "latest",
PushTime: time.Now(),
PullTime: time.Now(),
}
m.dao.On("Count", mock.Anything).Return(1, nil)
m.dao.On("List", mock.Anything).Return([]*tag.Tag{tg}, nil)
total, tags, err := m.mgr.List(nil, nil)
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
m.Equal(int64(1), total)
m.Equal(1, len(tags))
m.Equal(tg.ID, tags[0].ID)
}
func (m *managerTestSuite) TestGet() {
m.dao.On("Get", mock.Anything).Return(&tag.Tag{}, nil)
_, err := m.mgr.Get(nil, 1)
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
}
func (m *managerTestSuite) TestCreate() {
m.dao.On("Create", mock.Anything).Return(1, nil)
_, err := m.mgr.Create(nil, nil)
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
}
func (m *managerTestSuite) TestUpdate() {
m.dao.On("Update", mock.Anything).Return(nil)
err := m.mgr.Update(nil, nil)
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
}
func (m *managerTestSuite) TestDelete() {
m.dao.On("Delete", mock.Anything).Return(nil)
err := m.mgr.Delete(nil, 1)
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
}
func TestManager(t *testing.T) {
suite.Run(t, &managerTestSuite{})
}

53
src/server/error/error.go Normal file
View File

@ -0,0 +1,53 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package error
import (
"errors"
ierror "github.com/goharbor/harbor/src/internal/error"
"net/http"
)
var (
codeMap = map[string]int{
ierror.BadRequestCode: http.StatusBadRequest,
ierror.UnAuthorizedCode: http.StatusUnauthorized,
ierror.ForbiddenCode: http.StatusForbidden,
ierror.NotFoundCode: http.StatusNotFound,
ierror.ConflictCode: http.StatusConflict,
ierror.PreconditionCode: http.StatusPreconditionFailed,
ierror.GeneralCode: http.StatusInternalServerError,
}
)
// APIError generates the HTTP status code and error payload based on the input err
func APIError(err error) (int, string) {
var e *ierror.Error
statusCode := 0
if errors.As(err, &e) {
statusCode = getHTTPStatusCode(e.Code)
} else {
statusCode = http.StatusInternalServerError
}
return statusCode, ierror.NewErrs(err).Error()
}
func getHTTPStatusCode(errCode string) int {
statusCode, ok := codeMap[errCode]
if !ok {
statusCode = http.StatusInternalServerError
}
return statusCode
}

53
src/server/error/error_test.go Normal file
View File

@ -0,0 +1,53 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package error
import (
"errors"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/stretchr/testify/assert"
"net/http"
"testing"
)
func TestGetHTTPStatusCode(t *testing.T) {
// pre-defined error code
errCode := ierror.NotFoundCode
statusCode := getHTTPStatusCode(errCode)
assert.Equal(t, http.StatusNotFound, statusCode)
// not-defined error code
errCode = "NOT_DEFINED_ERROR_CODE"
statusCode = getHTTPStatusCode(errCode)
assert.Equal(t, http.StatusInternalServerError, statusCode)
}
func TestAPIError(t *testing.T) {
// ierror.Error
err := &ierror.Error{
Cause: nil,
Code: ierror.NotFoundCode,
Message: "resource not found",
}
statusCode, payload := APIError(err)
assert.Equal(t, http.StatusNotFound, statusCode)
assert.Equal(t, `{"errors":[{"code":"NOT_FOUND","message":"resource not found"}]}`, payload)
// common error
e := errors.New("customized error")
statusCode, payload = APIError(e)
assert.Equal(t, http.StatusInternalServerError, statusCode)
assert.Equal(t, `{"errors":[{"code":"UNKNOWN","message":"customized error"}]}`, payload)
}

0
src/server/v2.0/registry/catalog/catalog.go → src/server/registry/catalog/catalog.go
View File

29
src/server/registry/error/error.go Normal file
View File

@ -0,0 +1,29 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package error
import (
"github.com/goharbor/harbor/src/common/utils/log"
serror "github.com/goharbor/harbor/src/server/error"
"net/http"
)
// Handle generates the HTTP status code and error payload and writes them to the response
func Handle(w http.ResponseWriter, req *http.Request, err error) {
log.Errorf("failed to handle the request %s: %v", req.URL.Path, err)
statusCode, payload := serror.APIError(err)
w.WriteHeader(statusCode)
w.Write([]byte(payload))
}

15
src/server/v2.0/registry/handler.go → src/server/registry/handler.go
View File

@ -15,13 +15,14 @@
package registry
import (
"github.com/goharbor/harbor/src/pkg/project"
"net/http"
"net/http/httputil"
"net/url"
"github.com/goharbor/harbor/src/server/v2.0/registry/catalog"
"github.com/goharbor/harbor/src/server/v2.0/registry/manifest"
"github.com/goharbor/harbor/src/server/v2.0/registry/tag"
"github.com/goharbor/harbor/src/server/registry/catalog"
"github.com/goharbor/harbor/src/server/registry/manifest"
"github.com/goharbor/harbor/src/server/registry/tag"
"github.com/gorilla/mux"
)
@ -39,17 +40,17 @@ func New(url *url.URL) http.Handler {
rootRouter.Path("/v2/_catalog").Methods(http.MethodGet).Handler(catalog.NewHandler())
// handle list tag
rootRouter.Path("/v2/{name}/tags/list").Methods(http.MethodGet).Handler(tag.NewHandler())
rootRouter.Path("/v2/{name:.*}/tags/list").Methods(http.MethodGet).Handler(tag.NewHandler())
// handle manifest
// TODO maybe we should split it into several sub routers based on the method
manifestRouter := rootRouter.Path("/v2/{name}/manifests/{reference}").Subrouter()
manifestRouter := rootRouter.Path("/v2/{name:.*}/manifests/{reference}").Subrouter()
manifestRouter.NewRoute().Methods(http.MethodGet, http.MethodHead, http.MethodPut, http.MethodDelete).
Handler(manifest.NewHandler(proxy))
Handler(manifest.NewHandler(project.Mgr, proxy))
// handle blob
// as we need to apply middleware to the blob requests, so create a sub router to handle the blob APIs
blobRouter := rootRouter.Path("/v2/{name}/blobs/").Subrouter()
blobRouter := rootRouter.PathPrefix("/v2/{name:.*}/blobs/").Subrouter()
blobRouter.NewRoute().Methods(http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete).
Handler(proxy)

135
src/server/registry/manifest/manifest.go Normal file
View File

@ -0,0 +1,135 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package manifest
import (
"github.com/goharbor/harbor/src/api/artifact"
"github.com/goharbor/harbor/src/api/repository"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/internal"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/project"
"github.com/goharbor/harbor/src/server/registry/error"
"github.com/gorilla/mux"
"github.com/opencontainers/go-digest"
"net/http"
"net/http/httputil"
)
// NewHandler returns the handler to handler manifest requests
func NewHandler(proMgr project.Manager, proxy *httputil.ReverseProxy) http.Handler {
return &handler{
proMgr: proMgr,
proxy: proxy,
}
}
type handler struct {
proMgr project.Manager
proxy *httputil.ReverseProxy
}
func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
switch req.Method {
case http.MethodHead:
h.head(w, req)
case http.MethodGet:
h.get(w, req)
case http.MethodDelete:
h.delete(w, req)
case http.MethodPut:
h.put(w, req)
}
}
// make sure the artifact exist before proxying the request to the backend registry
func (h *handler) head(w http.ResponseWriter, req *http.Request) {
// TODO check the existence
h.proxy.ServeHTTP(w, req)
}
// make sure the artifact exist before proxying the request to the backend registry
func (h *handler) get(w http.ResponseWriter, req *http.Request) {
// TODO check the existence
h.proxy.ServeHTTP(w, req)
}
func (h *handler) delete(w http.ResponseWriter, req *http.Request) {
// TODO implement, just delete from database
}
func (h *handler) put(w http.ResponseWriter, req *http.Request) {
vars := mux.Vars(req)
repositoryName := vars["name"]
projectName, _ := utils.ParseRepository(repositoryName)
project, err := h.proMgr.Get(projectName)
if err != nil {
error.Handle(w, req, err)
return
}
if project == nil {
error.Handle(w, req,
ierror.NotFoundError(nil).WithMessage("project %s not found", projectName))
return
}
// make sure the repository exist before pushing the manifest
_, repositoryID, err := repository.Ctl.Ensure(req.Context(), project.ProjectID, repositoryName)
if err != nil {
error.Handle(w, req, err)
return
}
buffer := internal.NewResponseBuffer(w)
// proxy the req to the backend docker registry
h.proxy.ServeHTTP(buffer, req)
if !buffer.Success() {
if _, err := buffer.Flush(); err != nil {
log.Errorf("failed to flush: %v", err)
}
return
}
// When got the response from the backend docker registry, the manifest and
// tag are both ready, so we don't need to handle the issue anymore:
// https://github.com/docker/distribution/issues/2625
var tags []string
var dgt string
reference := vars["reference"]
dg, err := digest.Parse(reference)
if err == nil {
// the reference is digest
dgt = dg.String()
} else {
// the reference is tag, get the digest from the response header
dgt = buffer.Header().Get("Docker-Content-Digest")
tags = append(tags, reference)
}
_, _, err = artifact.Ctl.Ensure(req.Context(), repositoryID, dgt, tags...)
if err != nil {
error.Handle(w, req, err)
return
}
// flush the origin response from the docker registry to the underlying response writer
if _, err := buffer.Flush(); err != nil {
log.Errorf("failed to flush: %v", err)
}
// TODO fire event, add access log in the event handler
}

17
src/server/registry/manifest/manifest_test.go Normal file
View File

@ -0,0 +1,17 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package manifest
// TODO

0
src/server/v2.0/registry/tag/tag.go → src/server/registry/tag/tag.go
View File

64
src/server/v2.0/registry/manifest/manifest.go
View File

@ -1,64 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package manifest
import (
"net/http"
"net/http/httputil"
)
// NewHandler returns the handler to handler manifest requests
func NewHandler(proxy *httputil.ReverseProxy) http.Handler {
return &handler{
proxy: proxy,
}
}
type handler struct {
proxy *httputil.ReverseProxy
}
func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
switch req.Method {
case http.MethodHead:
h.head(w, req)
case http.MethodGet:
h.get(w, req)
case http.MethodDelete:
h.delete(w, req)
case http.MethodPut:
h.put(w, req)
}
}
// make sure the artifact exist before proxying the request to the backend registry
func (h *handler) head(w http.ResponseWriter, req *http.Request) {
// TODO check the existence
h.proxy.ServeHTTP(w, req)
}
// make sure the artifact exist before proxying the request to the backend registry
func (h *handler) get(w http.ResponseWriter, req *http.Request) {
// TODO check the existence
h.proxy.ServeHTTP(w, req)
}
func (h *handler) delete(w http.ResponseWriter, req *http.Request) {
// TODO implement, just delete from database
}
func (h *handler) put(w http.ResponseWriter, req *http.Request) {
h.proxy.ServeHTTP(w, req)
}

58
src/testing/artifact_manager.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package testing
import (
"context"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/stretchr/testify/mock"
"time"
)
// FakeArtifactManager is a fake artifact manager that implement src/pkg/artifact.Manager interface
type FakeArtifactManager struct {
mock.Mock
}
// List ...
func (f *FakeArtifactManager) List(ctx context.Context, query *q.Query) (int64, []*artifact.Artifact, error) {
args := f.Called()
return int64(args.Int(0)), args.Get(1).([]*artifact.Artifact), args.Error(2)
}
// Get ...
func (f *FakeArtifactManager) Get(ctx context.Context, id int64) (*artifact.Artifact, error) {
args := f.Called()
return args.Get(0).(*artifact.Artifact), args.Error(1)
}
// Create ...
func (f *FakeArtifactManager) Create(ctx context.Context, artifact *artifact.Artifact) (int64, error) {
args := f.Called()
return int64(args.Int(0)), args.Error(1)
}
// Delete ...
func (f *FakeArtifactManager) Delete(ctx context.Context, id int64) error {
args := f.Called()
return args.Error(0)
}
// UpdatePullTime ...
func (f *FakeArtifactManager) UpdatePullTime(ctx context.Context, artifactID int64, time time.Time) error {
args := f.Called()
return args.Error(0)
}

57
src/testing/repository_manager.go Normal file
View File

@ -0,0 +1,57 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package testing
import (
"context"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/stretchr/testify/mock"
)
// FakeRepositoryManager is a fake repository manager that implement src/pkg/repository.Manager interface
type FakeRepositoryManager struct {
mock.Mock
}
// List ...
func (f *FakeRepositoryManager) List(ctx context.Context, query *q.Query) (int64, []*models.RepoRecord, error) {
args := f.Called()
return int64(args.Int(0)), args.Get(1).([]*models.RepoRecord), args.Error(2)
}
// Get ...
func (f *FakeRepositoryManager) Get(ctx context.Context, id int64) (*models.RepoRecord, error) {
args := f.Called()
return args.Get(0).(*models.RepoRecord), args.Error(1)
}
// Delete ...
func (f *FakeRepositoryManager) Delete(ctx context.Context, id int64) error {
args := f.Called()
return args.Error(0)
}
// Create ...
func (f *FakeRepositoryManager) Create(ctx context.Context, repository *models.RepoRecord) (int64, error) {
args := f.Called()
return int64(args.Int(0)), args.Error(1)
}
// Update ...
func (f *FakeRepositoryManager) Update(ctx context.Context, repository *models.RepoRecord, props ...string) error {
args := f.Called()
return args.Error(0)
}

57
src/testing/tag_manager.go Normal file
View File

@ -0,0 +1,57 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package testing
import (
"context"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/tag/model/tag"
"github.com/stretchr/testify/mock"
)
// FakeTagManager is a fake tag manager that implement the src/pkg/tag.Manager interface
type FakeTagManager struct {
mock.Mock
}
// List ...
func (f *FakeTagManager) List(ctx context.Context, query *q.Query) (int64, []*tag.Tag, error) {
args := f.Called()
return int64(args.Int(0)), args.Get(1).([]*tag.Tag), args.Error(2)
}
// Get ...
func (f *FakeTagManager) Get(ctx context.Context, id int64) (*tag.Tag, error) {
args := f.Called()
return args.Get(0).(*tag.Tag), args.Error(1)
}
// Create ...
func (f *FakeTagManager) Create(ctx context.Context, tag *tag.Tag) (int64, error) {
args := f.Called()
return int64(args.Int(0)), args.Error(1)
}
// Update ...
func (f *FakeTagManager) Update(ctx context.Context, tag *tag.Tag, props ...string) error {
args := f.Called()
return args.Error(0)
}
// Delete ...
func (f *FakeTagManager) Delete(ctx context.Context, id int64) error {
args := f.Called()
return args.Error(0)
}