Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2025-01-03 06:28:06 +01:00
Code Issues Projects Releases Wiki Activity

feat(middleware): add blob middlewares (#10710)

Browse Source 
1. Add middleware to record the accepted blob size for stream blob
upload.
2. Add middleware to create blob and associate it with project after blob upload
complete.
3. Add middleware to sync blobs, create blob for manifest and associate blobs
with the manifest after put manifest.
4. Add middleware to associate blob with project after mount blob.
5. Cleanup associations for the project when artifact deleted.

Signed-off-by: He Weiwei <hweiwei@vmware.com>
This commit is contained in:
He Weiwei 2020-02-20 23:20:34 +08:00 committed by GitHub
parent 902e4826a6
commit 88fcacd4b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 3148 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -426,7 +426,7 @@ gofmt:
commentfmt:
@echo checking comment format...
@res=$$(find . -type d \( -path ./src/vendor -o -path ./tests \) -prune -o -name '*.go' -print | xargs egrep '(^|\s)\/\/(\S)'); \
@res=$$(find . -type d \( -path ./src/vendor -o -path ./tests \) -prune -o -name '*.go' -print | xargs grep -P '(^|\s)\/\/(?!go:generate\s)(\S)'); \
if [ -n "$${res}" ]; then \
echo checking comment format fail.. ; \
echo missing whitespace between // and comment body;\

19
src/api/artifact/controller.go
View File

@ -17,6 +17,9 @@ package artifact
import (
"context"
"fmt"
"strings"
"time"
"github.com/goharbor/harbor/src/api/artifact/abstractor"
"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
"github.com/goharbor/harbor/src/api/artifact/descriptor"
@ -26,13 +29,14 @@ import (
"github.com/goharbor/harbor/src/pkg/art"
"github.com/goharbor/harbor/src/pkg/artifactrash"
"github.com/goharbor/harbor/src/pkg/artifactrash/model"
"github.com/goharbor/harbor/src/pkg/blob"
"github.com/goharbor/harbor/src/pkg/immutabletag/match"
"github.com/goharbor/harbor/src/pkg/immutabletag/match/rule"
"github.com/goharbor/harbor/src/pkg/label"
"github.com/goharbor/harbor/src/pkg/registry"
"github.com/goharbor/harbor/src/pkg/signature"
"github.com/opencontainers/go-digest"
"strings"
// registry image resolvers
_ "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver/image"
// register chart resolver
@ -44,7 +48,6 @@ import (
"github.com/goharbor/harbor/src/pkg/repository"
"github.com/goharbor/harbor/src/pkg/tag"
tm "github.com/goharbor/harbor/src/pkg/tag/model/tag"
"time"
)
var (
@ -99,6 +102,7 @@ func NewController() Controller {
repoMgr: repository.Mgr,
artMgr: artifact.Mgr,
artrashMgr: artifactrash.Mgr,
blobMgr: blob.Mgr,
tagMgr: tag.Mgr,
sigMgr: signature.GetManager(),
labelMgr: label.Mgr,
@ -114,6 +118,7 @@ type controller struct {
repoMgr repository.Manager
artMgr artifact.Manager
artrashMgr artifactrash.Manager
blobMgr blob.Manager
tagMgr tag.Manager
sigMgr signature.Manager
labelMgr label.Manager
@ -352,6 +357,16 @@ func (c *controller) deleteDeeply(ctx context.Context, id int64, isRoot bool) er
return err
}
blobs, err := c.blobMgr.List(ctx, blob.ListParams{ArtifactDigest: art.Digest})
if err != nil {
return err
}
// clean associations between blob and project when when the blob is not needed by project
if err := c.blobMgr.CleanupAssociationsForProject(ctx, art.ProjectID, blobs); err != nil {
return err
}
// delete the artifact itself
if err = c.artMgr.Delete(ctx, art.ID); err != nil {
// the child artifact doesn't exist, skip

11
src/api/artifact/controller_test.go
View File

@ -16,6 +16,9 @@ package artifact
import (
"context"
"testing"
"time"
"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
"github.com/goharbor/harbor/src/api/artifact/descriptor"
"github.com/goharbor/harbor/src/common/models"
@ -26,6 +29,7 @@ import (
"github.com/goharbor/harbor/src/pkg/tag/model/tag"
arttesting "github.com/goharbor/harbor/src/testing/pkg/artifact"
artrashtesting "github.com/goharbor/harbor/src/testing/pkg/artifactrash"
"github.com/goharbor/harbor/src/testing/pkg/blob"
immutesting "github.com/goharbor/harbor/src/testing/pkg/immutabletag"
"github.com/goharbor/harbor/src/testing/pkg/label"
"github.com/goharbor/harbor/src/testing/pkg/registry"
@ -33,8 +37,6 @@ import (
tagtesting "github.com/goharbor/harbor/src/testing/pkg/tag"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"testing"
"time"
)
type fakeAbstractor struct {
@ -72,6 +74,7 @@ type controllerTestSuite struct {
repoMgr *repotesting.FakeManager
artMgr *arttesting.FakeManager
artrashMgr *artrashtesting.FakeManager
blobMgr *blob.Manager
tagMgr *tagtesting.FakeManager
labelMgr *label.FakeManager
abstractor *fakeAbstractor
@ -83,6 +86,7 @@ func (c *controllerTestSuite) SetupTest() {
c.repoMgr = &repotesting.FakeManager{}
c.artMgr = &arttesting.FakeManager{}
c.artrashMgr = &artrashtesting.FakeManager{}
c.blobMgr = &blob.Manager{}
c.tagMgr = &tagtesting.FakeManager{}
c.labelMgr = &label.FakeManager{}
c.abstractor = &fakeAbstractor{}
@ -92,6 +96,7 @@ func (c *controllerTestSuite) SetupTest() {
repoMgr: c.repoMgr,
artMgr: c.artMgr,
artrashMgr: c.artrashMgr,
blobMgr: c.blobMgr,
tagMgr: c.tagMgr,
labelMgr: c.labelMgr,
abstractor: c.abstractor,
@ -485,6 +490,8 @@ func (c *controllerTestSuite) TestDeleteDeeply() {
// root artifact is referenced by other artifacts
c.artMgr.On("Get").Return(&artifact.Artifact{ID: 1}, nil)
c.tagMgr.On("List").Return(nil, nil)
c.blobMgr.On("List", nil, mock.AnythingOfType("models.ListParams")).Return(nil, nil).Once()
c.blobMgr.On("CleanupAssociationsForProject", nil, int64(0), mock.AnythingOfType("[]*models.Blob")).Return(nil).Once()
c.repoMgr.On("Get").Return(&models.RepoRecord{}, nil)
c.artMgr.On("ListReferences").Return(nil, nil)
c.tagMgr.On("DeleteOfArtifact").Return(nil)

279
src/api/blob/controller.go Normal file
View File

@ -0,0 +1,279 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"context"
"fmt"
"github.com/docker/distribution"
"github.com/garyburd/redigo/redis"
"github.com/goharbor/harbor/src/common/utils/log"
util "github.com/goharbor/harbor/src/common/utils/redis"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/internal/orm"
"github.com/goharbor/harbor/src/pkg/blob"
)
var (
// Ctl is a global blob controller instance
Ctl = NewController()
)
// Controller defines the operations related with blobs
type Controller interface {
// AssociateWithArtifact associate blobs with manifest.
AssociateWithArtifact(ctx context.Context, blobDigests []string, artifactDigest string) error
// AssociateWithProjectByID associate blob with project by blob id
AssociateWithProjectByID(ctx context.Context, blobID int64, projectID int64) error
// AssociateWithProjectByDigest associate blob with project by blob digest
AssociateWithProjectByDigest(ctx context.Context, blobDigest string, projectID int64) error
// Ensure create blob when it not exist.
Ensure(ctx context.Context, digest string, contentType string, size int64) (int64, error)
// Exist check blob exist by digest,
// it check the blob associated with the artifact when `IsAssociatedWithArtifact` option provided,
// and also check the blob associated with the project when `IsAssociatedWithProject` option provied.
Exist(ctx context.Context, digest string, options ...Option) (bool, error)
// Get get the blob by digest,
// it check the blob associated with the artifact when `IsAssociatedWithArtifact` option provided,
// and also check the blob associated with the project when `IsAssociatedWithProject` option provied.
Get(ctx context.Context, digest string, options ...Option) (*blob.Blob, error)
// Sync create blobs from `References` when they are not exist
// and update the blob content type when they are exist,
Sync(ctx context.Context, references []distribution.Descriptor) error
// SetAcceptedBlobSize update the accepted size of stream upload blob.
SetAcceptedBlobSize(sessionID string, size int64) error
// GetAcceptedBlobSize returns the accepted size of stream upload blob.
GetAcceptedBlobSize(sessionID string) (int64, error)
}
// NewController creates an instance of the default repository controller
func NewController() Controller {
return &controller{
blobMgr: blob.Mgr,
logPrefix: "[controller][blob]",
}
}
type controller struct {
blobMgr blob.Manager
logPrefix string
}
func (c *controller) AssociateWithArtifact(ctx context.Context, blobDigests []string, artifactDigest string) error {
exist, err := c.blobMgr.IsAssociatedWithArtifact(ctx, artifactDigest, artifactDigest)
if err != nil {
return err
}
if exist {
log.Infof("%s: artifact digest %s already exist, skip to associate blobs with the artifact", c.logPrefix, artifactDigest)
return nil
}
for _, blobDigest := range blobDigests {
_, err := c.blobMgr.AssociateWithArtifact(ctx, blobDigest, artifactDigest)
if err != nil {
return err
}
}
// process manifest as blob
_, err = c.blobMgr.AssociateWithArtifact(ctx, artifactDigest, artifactDigest)
return err
}
func (c *controller) AssociateWithProjectByID(ctx context.Context, blobID int64, projectID int64) error {
_, err := c.blobMgr.AssociateWithProject(ctx, blobID, projectID)
return err
}
func (c *controller) AssociateWithProjectByDigest(ctx context.Context, blobDigest string, projectID int64) error {
blob, err := c.blobMgr.Get(ctx, blobDigest)
if err != nil {
return err
}
_, err = c.blobMgr.AssociateWithProject(ctx, blob.ID, projectID)
return err
}
func (c *controller) Get(ctx context.Context, digest string, options ...Option) (*blob.Blob, error) {
if digest == "" {
return nil, ierror.New(nil).WithCode(ierror.BadRequestCode).WithMessage("require Digest")
}
blob, err := c.blobMgr.Get(ctx, digest)
if err != nil {
return nil, err
}
opts := &Options{}
for _, f := range options {
f(opts)
}
if opts.ProjectID != 0 {
exist, err := c.blobMgr.IsAssociatedWithProject(ctx, digest, opts.ProjectID)
if err != nil {
return nil, err
}
if !exist {
return nil, ierror.NotFoundError(nil).WithMessage("blob %s is not associated with the project %d", digest, opts.ProjectID)
}
}
if opts.ArtifactDigest != "" {
exist, err := c.blobMgr.IsAssociatedWithArtifact(ctx, digest, opts.ArtifactDigest)
if err != nil {
return nil, err
}
if !exist {
return nil, ierror.NotFoundError(nil).WithMessage("blob %s is not associated with the artifact %s", digest, opts.ArtifactDigest)
}
}
return blob, nil
}
func (c *controller) Ensure(ctx context.Context, digest string, contentType string, size int64) (blobID int64, err error) {
blob, err := c.blobMgr.Get(ctx, digest)
if err == nil {
return blob.ID, nil
}
if !ierror.IsNotFoundErr(err) {
return 0, err
}
return c.blobMgr.Create(ctx, digest, contentType, size)
}
func (c *controller) Exist(ctx context.Context, digest string, options ...Option) (bool, error) {
if digest == "" {
return false, ierror.BadRequestError(nil).WithMessage("exist blob require digest")
}
_, err := c.Get(ctx, digest, options...)
if err != nil {
if ierror.IsNotFoundErr(err) {
return false, nil
}
return false, err
}
return true, nil
}
func (c *controller) Sync(ctx context.Context, references []distribution.Descriptor) error {
if len(references) == 0 {
return nil
}
var digests []string
for _, reference := range references {
digests = append(digests, reference.Digest.String())
}
blobs, err := c.blobMgr.List(ctx, blob.ListParams{BlobDigests: digests})
if err != nil {
return err
}
mp := make(map[string]*blob.Blob, len(blobs))
for _, blob := range blobs {
mp[blob.Digest] = blob
}
var missing, updating []*blob.Blob
for _, reference := range references {
if exist, found := mp[reference.Digest.String()]; found {
if exist.ContentType != reference.MediaType {
exist.ContentType = reference.MediaType
updating = append(updating, exist)
}
} else {
missing = append(missing, &blob.Blob{
Digest: reference.Digest.String(),
ContentType: reference.MediaType,
Size: reference.Size,
})
}
}
if len(updating) > 0 {
orm.WithTransaction(func(ctx context.Context) error {
for _, blob := range updating {
if err := c.blobMgr.Update(ctx, blob); err != nil {
log.Warningf("Failed to update blob %s, error: %v", blob.Digest, err)
return err
}
}
return nil
})(ctx)
}
if len(missing) > 0 {
for _, blob := range missing {
if _, err := c.blobMgr.Create(ctx, blob.Digest, blob.ContentType, blob.Size); err != nil {
return err
}
}
}
return nil
}
func (c *controller) SetAcceptedBlobSize(sessionID string, size int64) error {
conn := util.DefaultPool().Get()
defer conn.Close()
key := fmt.Sprintf("upload:%s:size", sessionID)
reply, err := redis.String(conn.Do("SET", key, size))
if err != nil {
return err
}
if reply != "OK" {
return fmt.Errorf("bad reply value")
}
return nil
}
func (c *controller) GetAcceptedBlobSize(sessionID string) (int64, error) {
conn := util.DefaultPool().Get()
defer conn.Close()
key := fmt.Sprintf("upload:%s:size", sessionID)
size, err := redis.Int64(conn.Do("GET", key))
if err != nil {
return 0, err
}
return size, nil
}

199
src/api/blob/controller_test.go Normal file
View File

@ -0,0 +1,199 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"fmt"
"testing"
"github.com/goharbor/harbor/src/pkg/distribution"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/google/uuid"
"github.com/stretchr/testify/suite"
)
type ControllerTestSuite struct {
htesting.Suite
}
func (suite *ControllerTestSuite) prepareBlob() string {
ctx := suite.Context()
digest := suite.DigestString()
_, err := Ctl.Ensure(ctx, digest, "application/octet-stream", 100)
suite.Nil(err)
return digest
}
func (suite *ControllerTestSuite) TestAttachToArtifact() {
ctx := suite.Context()
artifactDigest := suite.DigestString()
blobDigests := []string{
suite.prepareBlob(),
suite.prepareBlob(),
suite.prepareBlob(),
}
suite.Nil(Ctl.AssociateWithArtifact(ctx, blobDigests, artifactDigest))
for _, digest := range blobDigests {
exist, err := Ctl.Exist(ctx, digest, IsAssociatedWithArtifact(artifactDigest))
suite.Nil(err)
suite.True(exist)
}
suite.Nil(Ctl.AssociateWithArtifact(ctx, blobDigests, artifactDigest))
}
func (suite *ControllerTestSuite) TestAttachToProjectByDigest() {
suite.WithProject(func(projectID int64, projectName string) {
ctx := suite.Context()
digest := suite.prepareBlob()
suite.Nil(Ctl.AssociateWithProjectByDigest(ctx, digest, projectID))
exist, err := Ctl.Exist(ctx, digest, IsAssociatedWithProject(projectID))
suite.Nil(err)
suite.True(exist)
})
}
func (suite *ControllerTestSuite) TestEnsure() {
ctx := suite.Context()
digest := suite.DigestString()
_, err := Ctl.Ensure(ctx, digest, "application/octet-stream", 100)
suite.Nil(err)
exist, err := Ctl.Exist(ctx, digest)
suite.Nil(err)
suite.True(exist)
_, err = Ctl.Ensure(ctx, digest, "application/octet-stream", 100)
suite.Nil(err)
}
func (suite *ControllerTestSuite) TestExist() {
ctx := suite.Context()
exist, err := Ctl.Exist(ctx, suite.DigestString())
suite.Nil(err)
suite.False(exist)
}
func (suite *ControllerTestSuite) TestGet() {
ctx := suite.Context()
{
digest := suite.prepareBlob()
blob, err := Ctl.Get(ctx, digest)
suite.Nil(err)
suite.Equal(digest, blob.Digest)
suite.Equal(int64(100), blob.Size)
suite.Equal("application/octet-stream", blob.ContentType)
}
{
digest := suite.prepareBlob()
artifactDigest := suite.DigestString()
_, err := Ctl.Get(ctx, digest, IsAssociatedWithArtifact(artifactDigest))
suite.NotNil(err)
Ctl.AssociateWithArtifact(ctx, []string{digest}, artifactDigest)
blob, err := Ctl.Get(ctx, digest, IsAssociatedWithArtifact(artifactDigest))
suite.Nil(err)
suite.Equal(digest, blob.Digest)
suite.Equal(int64(100), blob.Size)
suite.Equal("application/octet-stream", blob.ContentType)
}
{
digest := suite.prepareBlob()
suite.WithProject(func(projectID int64, projectName string) {
_, err := Ctl.Get(ctx, digest, IsAssociatedWithProject(projectID))
suite.NotNil(err)
Ctl.AssociateWithProjectByDigest(ctx, digest, projectID)
blob, err := Ctl.Get(ctx, digest, IsAssociatedWithProject(projectID))
suite.Nil(err)
suite.Equal(digest, blob.Digest)
suite.Equal(int64(100), blob.Size)
suite.Equal("application/octet-stream", blob.ContentType)
})
}
}
func (suite *ControllerTestSuite) TestSync() {
var references []distribution.Descriptor
for i := 0; i < 5; i++ {
references = append(references, distribution.Descriptor{
MediaType: fmt.Sprintf("media type %d", i),
Digest: suite.Digest(),
Size: int64(100 + i),
})
}
suite.WithProject(func(projectID int64, projectName string) {
ctx := suite.Context()
{
suite.Nil(Ctl.Sync(ctx, references))
for _, reference := range references {
blob, err := Ctl.Get(ctx, reference.Digest.String())
suite.Nil(err)
suite.Equal(reference.MediaType, blob.ContentType)
suite.Equal(reference.Digest.String(), blob.Digest)
suite.Equal(reference.Size, blob.Size)
}
}
{
references[0].MediaType = "media type"
references = append(references, distribution.Descriptor{
MediaType: "media type",
Digest: suite.Digest(),
Size: int64(100),
})
suite.Nil(Ctl.Sync(ctx, references))
}
})
}
func (suite *ControllerTestSuite) TestGetSetAcceptedBlobSize() {
sessionID := uuid.New().String()
size, err := Ctl.GetAcceptedBlobSize(sessionID)
suite.NotNil(err)
suite.Nil(Ctl.SetAcceptedBlobSize(sessionID, 100))
size, err = Ctl.GetAcceptedBlobSize(sessionID)
suite.Nil(err)
suite.Equal(int64(100), size)
}
func TestControllerTestSuite(t *testing.T) {
suite.Run(t, &ControllerTestSuite{})
}

38
src/api/blob/options.go Normal file
View File

@ -0,0 +1,38 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
// Option option for `Get` and `Exist` method of `Controller`
type Option func(*Options)
// Options options used by `Get` method of `Controller`
type Options struct {
ArtifactDigest string // blob associated with the artifact
ProjectID int64 // blob associated with the project
}
// IsAssociatedWithArtifact set ArtifactDigest for the Options
func IsAssociatedWithArtifact(artifactDigest string) Option {
return func(opts *Options) {
opts.ArtifactDigest = artifactDigest
}
}
// IsAssociatedWithProject set ProjectID for the Options
func IsAssociatedWithProject(projectID int64) Option {
return func(opts *Options) {
opts.ProjectID = projectID
}
}

61
src/api/project/controller.go Normal file
View File

@ -0,0 +1,61 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package project
import (
"context"
"github.com/goharbor/harbor/src/common/models"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/project"
)
var (
// Ctl is a global project controller instance
Ctl = NewController()
)
// Controller defines the operations related with blobs
type Controller interface {
// GetByName get the project by project name
GetByName(ctx context.Context, projectName string) (*models.Project, error)
}
// NewController creates an instance of the default project controller
func NewController() Controller {
return &controller{
projectMgr: project.Mgr,
}
}
type controller struct {
projectMgr project.Manager
}
func (c *controller) GetByName(ctx context.Context, projectName string) (*models.Project, error) {
if projectName == "" {
return nil, ierror.BadRequestError(nil).WithMessage("project name required")
}
p, err := c.projectMgr.Get(projectName)
if err != nil {
return nil, err
}
if p == nil {
return nil, ierror.NotFoundError(nil).WithMessage("project %s not found", projectName)
}
return p, nil
}

64
src/api/project/controller_test.go Normal file
View File

@ -0,0 +1,64 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package project
import (
"context"
"fmt"
"testing"
"github.com/goharbor/harbor/src/common/models"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/testing/pkg/project"
"github.com/stretchr/testify/suite"
)
type ControllerTestSuite struct {
suite.Suite
}
func (suite *ControllerTestSuite) TestGetByName() {
mgr := &project.FakeManager{}
mgr.On("Get", "library").Return(&models.Project{ProjectID: 1, Name: "library"}, nil)
mgr.On("Get", "test").Return(nil, nil)
mgr.On("Get", "oops").Return(nil, fmt.Errorf("oops"))
c := controller{projectMgr: mgr}
{
p, err := c.GetByName(context.TODO(), "library")
suite.Nil(err)
suite.Equal("library", p.Name)
suite.Equal(int64(1), p.ProjectID)
}
{
p, err := c.GetByName(context.TODO(), "test")
suite.Error(err)
suite.True(ierror.IsNotFoundErr(err))
suite.Nil(p)
}
{
p, err := c.GetByName(context.TODO(), "oops")
suite.Error(err)
suite.False(ierror.IsNotFoundErr(err))
suite.Nil(p)
}
}
func TestControllerTestSuite(t *testing.T) {
suite.Run(t, &ControllerTestSuite{})
}

5
src/api/repository/controller_test.go
View File

@ -15,13 +15,14 @@
package repository
import (
"testing"
"github.com/goharbor/harbor/src/api/artifact"
"github.com/goharbor/harbor/src/common/models"
artifacttesting "github.com/goharbor/harbor/src/testing/api/artifact"
"github.com/goharbor/harbor/src/testing/pkg/project"
"github.com/goharbor/harbor/src/testing/pkg/repository"
"github.com/stretchr/testify/suite"
"testing"
)
type controllerTestSuite struct {
@ -63,7 +64,7 @@ func (c *controllerTestSuite) TestEnsure() {
// doesn't exist
c.repoMgr.On("List").Return([]*models.RepoRecord{}, nil)
c.proMgr.On("Get").Return(&models.Project{
c.proMgr.On("Get", "library").Return(&models.Project{
ProjectID: 1,
}, nil)
c.repoMgr.On("Create").Return(1, nil)

6
src/common/dao/artifact_blob_test.go
View File

@ -18,8 +18,6 @@ import (
"testing"
"github.com/goharbor/harbor/src/common/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -30,10 +28,8 @@ func TestAddArtifactNBlob(t *testing.T) {
}
// add
id, err := AddArtifactNBlob(afnb)
_, err := AddArtifactNBlob(afnb)
require.Nil(t, err)
afnb.ID = id
assert.Equal(t, id, int64(1))
}
func TestAddArtifactNBlobs(t *testing.T) {

5
src/internal/error/errors.go
View File

@ -155,6 +155,11 @@ func IsErr(err error, code string) bool {
return false
}
// IsNotFoundErr returns true when the error is NotFoundError
func IsNotFoundErr(err error) bool {
return IsErr(err, NotFoundCode)
}
// IsConflictErr checks whether the err chain contains conflict error
func IsConflictErr(err error) bool {
return IsErr(err, ConflictCode)

19
src/internal/orm/error.go
View File

@ -16,11 +16,30 @@ package orm
import (
"errors"
"github.com/astaxie/beego/orm"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/lib/pq"
)
// WrapNotFoundError wrap error as NotFoundError when it is orm.ErrNoRows otherwise return err
func WrapNotFoundError(err error, format string, args ...interface{}) error {
if e := AsNotFoundError(err, format, args...); e != nil {
return e
}
return err
}
// WrapConflictError wrap error as ConflictError when it is duplicate key error otherwise return err
func WrapConflictError(err error, format string, args ...interface{}) error {
if e := AsConflictError(err, format, args...); e != nil {
return e
}
return err
}
// AsNotFoundError checks whether the err is orm.ErrNoRows. If it it, wrap it
// as a src/internal/error.Error with not found error code, else return nil
func AsNotFoundError(err error, messageFormat string, args ...interface{}) *ierror.Error {

12
src/internal/orm/query.go
View File

@ -16,6 +16,8 @@ package orm
import (
"context"
"strings"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/pkg/q"
)
@ -41,3 +43,13 @@ func QuerySetter(ctx context.Context, model interface{}, query *q.Query) (orm.Qu
}
return qs, nil
}
// ParamPlaceholderForIn returns a string that contains placeholders for sql keyword "in"
// e.g. n=3, returns "?,?,?"
func ParamPlaceholderForIn(n int) string {
placeholders := []string{}
for i := 0; i < n; i++ {
placeholders = append(placeholders, "?")
}
return strings.Join(placeholders, ",")
}

66
src/internal/request.go Normal file
View File

@ -0,0 +1,66 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package internal
import (
"bytes"
"io"
"net/http"
)
// nopCloser is just like ioutil's, but here to let us re-read the same
// buffer inside by moving position to the start every time we done with reading
type nopCloser struct {
io.ReadSeeker
}
// Read just a wrapper around real Read which also moves position to the start if we get EOF
// to have it ready for next read-cycle
func (n nopCloser) Read(p []byte) (int, error) {
num, err := n.ReadSeeker.Read(p)
if err == io.EOF { // move to start to have it ready for next read cycle
n.Seek(0, io.SeekStart)
}
return num, err
}
// Close is a no-op Close
func (n nopCloser) Close() error {
return nil
}
func copyBody(body io.ReadCloser) io.ReadCloser {
// check if body was already read and converted into our nopCloser
if nc, ok := body.(nopCloser); ok {
nc.Seek(0, io.SeekStart)
return body
}
defer body.Close()
var buf bytes.Buffer
io.Copy(&buf, body)
return nopCloser{bytes.NewReader(buf.Bytes())}
}
// NopCloseRequest ...
func NopCloseRequest(r *http.Request) *http.Request {
if r != nil && r.Body != nil {
r.Body = copyBody(r.Body)
}
return r
}

55
src/internal/request_test.go Normal file
View File

@ -0,0 +1,55 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package internal
import (
"io/ioutil"
"net/http"
"strings"
"testing"
"github.com/stretchr/testify/suite"
)
type NopCloseRequestTestSuite struct {
suite.Suite
}
func (suite *NopCloseRequestTestSuite) TestReusableBody() {
r, _ := http.NewRequest(http.MethodPost, "/", strings.NewReader("body"))
body, err := ioutil.ReadAll(r.Body)
suite.Nil(err)
suite.Equal([]byte("body"), body)
body, err = ioutil.ReadAll(r.Body)
suite.Nil(err)
suite.Equal([]byte(""), body)
r, _ = http.NewRequest(http.MethodPost, "/", strings.NewReader("body"))
r = NopCloseRequest(r)
body, err = ioutil.ReadAll(r.Body)
suite.Nil(err)
suite.Equal([]byte("body"), body)
body, err = ioutil.ReadAll(r.Body)
suite.Nil(err)
suite.Equal([]byte("body"), body)
}
func TestNopCloseRequestTestSuite(t *testing.T) {
suite.Run(t, &NopCloseRequestTestSuite{})
}

5
src/internal/response_buffer.go
View File

@ -95,3 +95,8 @@ func (r *ResponseBuffer) Reset() error {
return nil
}
// StatusCode returns the status code
func (r *ResponseBuffer) StatusCode() int {
return r.code
}

265
src/pkg/blob/dao/dao.go Normal file
View File

@ -0,0 +1,265 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"context"
"fmt"
"time"
"github.com/goharbor/harbor/src/internal/orm"
"github.com/goharbor/harbor/src/pkg/blob/models"
"github.com/goharbor/harbor/src/pkg/q"
)
// DAO the dao for Blob, ArtifactAndBlob and ProjectBlob
type DAO interface {
// CreateArtifactAndBlob create ArtifactAndBlob and ignore conflict on artifact digest and blob digest
CreateArtifactAndBlob(ctx context.Context, artifactDigest, blobDigest string) (int64, error)
// GetArtifactAndBlob get ArtifactAndBlob by artifact digest and blob digest
GetArtifactAndBlob(ctx context.Context, artifactDigest, blobDigest string) (*models.ArtifactAndBlob, error)
// DeleteArtifactAndBlobByArtifact delete ArtifactAndBlob by artifact digest
DeleteArtifactAndBlobByArtifact(ctx context.Context, artifactDigest string) error
// GetAssociatedBlobDigestsForArtifact returns blob digests which associated with the artifact
GetAssociatedBlobDigestsForArtifact(ctx context.Context, artifact string) ([]string, error)
// CreateBlob create blob and ignore conflict on digest
CreateBlob(ctx context.Context, blob *models.Blob) (int64, error)
// GetBlobByDigest returns blob by digest
GetBlobByDigest(ctx context.Context, digest string) (*models.Blob, error)
// UpdateBlob update blob
UpdateBlob(ctx context.Context, blob *models.Blob) error
// ListBlobs list blobs by query
ListBlobs(ctx context.Context, query *q.Query) ([]*models.Blob, error)
// FindBlobsShouldUnassociatedWithProject filter the blobs which should not be associated with the project
FindBlobsShouldUnassociatedWithProject(ctx context.Context, projectID int64, blobs []*models.Blob) ([]*models.Blob, error)
// CreateProjectBlob create ProjectBlob and ignore conflict on project id and blob id
CreateProjectBlob(ctx context.Context, projectID, blobID int64) (int64, error)
// DeleteProjectBlob delete project blob
DeleteProjectBlob(ctx context.Context, projectID int64, blobIDs ...int64) error
// ExistProjectBlob returns true when ProjectBlob exist
ExistProjectBlob(ctx context.Context, projectID int64, blobDigest string) (bool, error)
}
// New returns an instance of the default DAO
func New() DAO {
return &dao{}
}
type dao struct{}
func (d *dao) CreateArtifactAndBlob(ctx context.Context, artifactDigest, blobDigest string) (int64, error) {
o, err := orm.FromContext(ctx)
if err != nil {
return 0, err
}
md := &models.ArtifactAndBlob{
DigestAF: artifactDigest,
DigestBlob: blobDigest,
CreationTime: time.Now(),
}
return o.InsertOrUpdate(md, "digest_af, digest_blob")
}
func (d *dao) GetArtifactAndBlob(ctx context.Context, artifactDigest, blobDigest string) (*models.ArtifactAndBlob, error) {
o, err := orm.FromContext(ctx)
if err != nil {
return nil, err
}
md := &models.ArtifactAndBlob{
DigestAF: artifactDigest,
DigestBlob: blobDigest,
}
if err := o.Read(md, "digest_af", "digest_blob"); err != nil {
return nil, orm.WrapNotFoundError(err, "not found by artifact digest %s and blob digest %s", artifactDigest, blobDigest)
}
return md, nil
}
func (d *dao) DeleteArtifactAndBlobByArtifact(ctx context.Context, artifactDigest string) error {
qs, err := orm.QuerySetter(ctx, &models.ArtifactAndBlob{}, q.New(q.KeyWords{"digest_af": artifactDigest}))
if err != nil {
return err
}
_, err = qs.Delete()
return err
}
func (d *dao) GetAssociatedBlobDigestsForArtifact(ctx context.Context, artifact string) ([]string, error) {
qs, err := orm.QuerySetter(ctx, &models.ArtifactAndBlob{}, q.New(q.KeyWords{"digest_af": artifact}))
if err != nil {
return nil, err
}
mds := []*models.ArtifactAndBlob{}
if _, err = qs.All(&mds); err != nil {
return nil, err
}
var blobDigests []string
for _, md := range mds {
blobDigests = append(blobDigests, md.DigestBlob)
}
return blobDigests, nil
}
func (d *dao) CreateBlob(ctx context.Context, blob *models.Blob) (int64, error) {
o, err := orm.FromContext(ctx)
if err != nil {
return 0, err
}
blob.CreationTime = time.Now()
return o.InsertOrUpdate(blob, "digest")
}
func (d *dao) GetBlobByDigest(ctx context.Context, digest string) (*models.Blob, error) {
o, err := orm.FromContext(ctx)
if err != nil {
return nil, err
}
blob := &models.Blob{Digest: digest}
if err = o.Read(blob, "digest"); err != nil {
return nil, orm.WrapNotFoundError(err, "blob %s not found", digest)
}
return blob, nil
}
func (d *dao) UpdateBlob(ctx context.Context, blob *models.Blob) error {
o, err := orm.FromContext(ctx)
if err != nil {
return err
}
_, err = o.Update(blob)
return err
}
func (d *dao) ListBlobs(ctx context.Context, query *q.Query) ([]*models.Blob, error) {
qs, err := orm.QuerySetter(ctx, &models.Blob{}, query)
if err != nil {
return nil, err
}
blobs := []*models.Blob{}
if _, err = qs.All(&blobs); err != nil {
return nil, err
}
return blobs, nil
}
func (d *dao) FindBlobsShouldUnassociatedWithProject(ctx context.Context, projectID int64, blobs []*models.Blob) ([]*models.Blob, error) {
if len(blobs) == 0 {
return nil, nil
}
o, err := orm.FromContext(ctx)
if err != nil {
return nil, err
}
sql := `SELECT b.digest_blob FROM artifact_2 a, artifact_blob b WHERE a.digest = b.digest_af AND a.project_id = ? AND b.digest_blob IN (%s)`
params := []interface{}{projectID}
for _, blob := range blobs {
params = append(params, blob.Digest)
}
var digests []string
_, err = o.Raw(fmt.Sprintf(sql, orm.ParamPlaceholderForIn(len(blobs))), params...).QueryRows(&digests)
if err != nil {
return nil, err
}
shouldAssociated := map[string]bool{}
for _, digest := range digests {
shouldAssociated[digest] = true
}
var results []*models.Blob
for _, blob := range blobs {
if !shouldAssociated[blob.Digest] {
results = append(results, blob)
}
}
return results, nil
}
func (d *dao) CreateProjectBlob(ctx context.Context, projectID, blobID int64) (int64, error) {
o, err := orm.FromContext(ctx)
if err != nil {
return 0, err
}
md := &models.ProjectBlob{
ProjectID: projectID,
BlobID: blobID,
CreationTime: time.Now(),
}
// ignore conflict error on (blob_id, project_id)
return o.InsertOrUpdate(md, "blob_id, project_id")
}
func (d *dao) ExistProjectBlob(ctx context.Context, projectID int64, blobDigest string) (bool, error) {
o, err := orm.FromContext(ctx)
if err != nil {
return false, err
}
sql := `SELECT COUNT(*) FROM project_blob JOIN blob ON project_blob.blob_id = blob.id AND project_id = ? AND digest = ?`
var count int64
if err := o.Raw(sql, projectID, blobDigest).QueryRow(&count); err != nil {
return false, err
}
return count > 0, nil
}
func (d *dao) DeleteProjectBlob(ctx context.Context, projectID int64, blobIDs ...int64) error {
if len(blobIDs) == 0 {
return nil
}
kw := q.KeyWords{"blob_id__in": blobIDs}
qs, err := orm.QuerySetter(ctx, &models.ProjectBlob{}, q.New(kw))
if err != nil {
return err
}
_, err = qs.Delete()
return err
}

278
src/pkg/blob/dao/dao_test.go Normal file
View File

@ -0,0 +1,278 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"testing"
"github.com/goharbor/harbor/src/pkg/blob/models"
"github.com/goharbor/harbor/src/pkg/q"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/stretchr/testify/suite"
)
type DaoTestSuite struct {
htesting.Suite
dao DAO
}
func (suite *DaoTestSuite) SetupSuite() {
suite.Suite.SetupSuite()
suite.Suite.ClearTables = []string{"blob", "artifact_blob", "project_blob"}
suite.dao = New()
}
func (suite *DaoTestSuite) TestCreateArtifactAndBlob() {
ctx := suite.Context()
artifactDigest := suite.DigestString()
blobDigest := suite.DigestString()
_, err := suite.dao.CreateArtifactAndBlob(ctx, artifactDigest, blobDigest)
suite.Nil(err)
_, err = suite.dao.CreateArtifactAndBlob(ctx, artifactDigest, blobDigest)
suite.Nil(err)
}
func (suite *DaoTestSuite) TestGetArtifactAndBlob() {
ctx := suite.Context()
artifactDigest := suite.DigestString()
blobDigest := suite.DigestString()
md, err := suite.dao.GetArtifactAndBlob(ctx, artifactDigest, blobDigest)
suite.IsNotFoundErr(err)
suite.Nil(md)
_, err = suite.dao.CreateArtifactAndBlob(ctx, artifactDigest, blobDigest)
suite.Nil(err)
md, err = suite.dao.GetArtifactAndBlob(ctx, artifactDigest, blobDigest)
if suite.Nil(err) {
suite.Equal(artifactDigest, md.DigestAF)
suite.Equal(blobDigest, md.DigestBlob)
}
}
func (suite *DaoTestSuite) TestDeleteArtifactAndBlobByArtifact() {
ctx := suite.Context()
artifactDigest := suite.DigestString()
blobDigest1 := suite.DigestString()
blobDigest2 := suite.DigestString()
_, err := suite.dao.CreateArtifactAndBlob(ctx, artifactDigest, blobDigest1)
suite.Nil(err)
_, err = suite.dao.CreateArtifactAndBlob(ctx, artifactDigest, blobDigest2)
suite.Nil(err)
digests, err := suite.dao.GetAssociatedBlobDigestsForArtifact(ctx, artifactDigest)
suite.Nil(err)
suite.Len(digests, 2)
suite.Nil(suite.dao.DeleteArtifactAndBlobByArtifact(ctx, artifactDigest))
digests, err = suite.dao.GetAssociatedBlobDigestsForArtifact(ctx, artifactDigest)
suite.Nil(err)
suite.Len(digests, 0)
}
func (suite *DaoTestSuite) TestGetAssociatedBlobDigestsForArtifact() {
}
func (suite *DaoTestSuite) TestCreateBlob() {
ctx := suite.Context()
digest := suite.DigestString()
_, err := suite.dao.CreateBlob(ctx, &models.Blob{Digest: digest})
suite.Nil(err)
_, err = suite.dao.CreateBlob(ctx, &models.Blob{Digest: digest})
suite.Nil(err)
}
func (suite *DaoTestSuite) TestGetBlobByDigest() {
ctx := suite.Context()
digest := suite.DigestString()
blob, err := suite.dao.GetBlobByDigest(ctx, digest)
suite.IsNotFoundErr(err)
suite.Nil(blob)
suite.dao.CreateBlob(ctx, &models.Blob{Digest: digest})
blob, err = suite.dao.GetBlobByDigest(ctx, digest)
if suite.Nil(err) {
suite.Equal(digest, blob.Digest)
}
}
func (suite *DaoTestSuite) TestUpdateBlob() {
ctx := suite.Context()
digest := suite.DigestString()
suite.dao.CreateBlob(ctx, &models.Blob{Digest: digest})
blob, err := suite.dao.GetBlobByDigest(ctx, digest)
if suite.Nil(err) {
suite.Equal(int64(0), blob.Size)
}
blob.Size = 100
if suite.Nil(suite.dao.UpdateBlob(ctx, blob)) {
blob, err := suite.dao.GetBlobByDigest(ctx, digest)
if suite.Nil(err) {
suite.Equal(int64(100), blob.Size)
}
}
}
func (suite *DaoTestSuite) TestListBlobs() {
ctx := suite.Context()
digest1 := suite.DigestString()
suite.dao.CreateBlob(ctx, &models.Blob{Digest: digest1})
digest2 := suite.DigestString()
suite.dao.CreateBlob(ctx, &models.Blob{Digest: digest2})
blobs, err := suite.dao.ListBlobs(ctx, q.New(q.KeyWords{"digest": digest1}))
if suite.Nil(err) {
suite.Len(blobs, 1)
}
blobs, err = suite.dao.ListBlobs(ctx, q.New(q.KeyWords{"digest__in": []string{digest1, digest2}}))
if suite.Nil(err) {
suite.Len(blobs, 2)
}
}
func (suite *DaoTestSuite) TestFindBlobsShouldUnassociatedWithProject() {
ctx := suite.Context()
suite.WithProject(func(projectID int64, projectName string) {
artifact1 := suite.DigestString()
artifact2 := suite.DigestString()
sql := `INSERT INTO artifact_2 ("type", media_type, manifest_media_type, digest, project_id, repository_id) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?)`
suite.ExecSQL(sql, artifact1, projectID, 10)
suite.ExecSQL(sql, artifact2, projectID, 10)
defer suite.ExecSQL(`DELETE FROM artifact_2 WHERE project_id = ?`, projectID)
digest1 := suite.DigestString()
digest2 := suite.DigestString()
digest3 := suite.DigestString()
digest4 := suite.DigestString()
digest5 := suite.DigestString()
blobDigests := []string{digest1, digest2, digest3, digest4, digest5}
for _, digest := range blobDigests {
blobID, err := suite.dao.CreateBlob(ctx, &models.Blob{Digest: digest})
if suite.Nil(err) {
suite.dao.CreateProjectBlob(ctx, projectID, blobID)
}
}
blobs, err := suite.dao.ListBlobs(ctx, q.New(q.KeyWords{"digest__in": blobDigests}))
suite.Nil(err)
suite.Len(blobs, 5)
for _, digest := range []string{digest1, digest2, digest3} {
suite.dao.CreateArtifactAndBlob(ctx, artifact1, digest)
}
for _, digest := range blobDigests {
suite.dao.CreateArtifactAndBlob(ctx, artifact2, digest)
}
{
results, err := suite.dao.FindBlobsShouldUnassociatedWithProject(ctx, projectID, blobs)
suite.Nil(err)
suite.Len(results, 0)
}
suite.ExecSQL(`DELETE FROM artifact_2 WHERE digest = ?`, artifact2)
{
results, err := suite.dao.FindBlobsShouldUnassociatedWithProject(ctx, projectID, blobs)
suite.Nil(err)
if suite.Len(results, 2) {
suite.Contains([]string{results[0].Digest, results[1].Digest}, digest4)
suite.Contains([]string{results[0].Digest, results[1].Digest}, digest5)
}
}
})
}
func (suite *DaoTestSuite) TestCreateProjectBlob() {
ctx := suite.Context()
projectID := int64(1)
blobID := int64(1000)
_, err := suite.dao.CreateProjectBlob(ctx, projectID, blobID)
suite.Nil(err)
_, err = suite.dao.CreateProjectBlob(ctx, projectID, blobID)
suite.Nil(err)
}
func (suite *DaoTestSuite) TestExistProjectBlob() {
ctx := suite.Context()
digest := suite.DigestString()
projectID := int64(1)
exist, err := suite.dao.ExistProjectBlob(ctx, projectID, digest)
suite.Nil(err)
suite.False(exist)
blobID, err := suite.dao.CreateBlob(ctx, &models.Blob{Digest: digest})
suite.Nil(err)
_, err = suite.dao.CreateProjectBlob(ctx, projectID, blobID)
suite.Nil(err)
exist, err = suite.dao.ExistProjectBlob(ctx, projectID, digest)
suite.Nil(err)
suite.True(exist)
}
func (suite *DaoTestSuite) TestDeleteProjectBlob() {
ctx := suite.Context()
projectID := int64(1)
blobID := int64(1000)
_, err := suite.dao.CreateProjectBlob(ctx, projectID, blobID)
suite.Nil(err)
suite.Nil(suite.dao.DeleteProjectBlob(ctx, projectID, blobID))
}
func TestDaoTestSuite(t *testing.T) {
suite.Run(t, &DaoTestSuite{})
}

161
src/pkg/blob/manager.go Normal file
View File

@ -0,0 +1,161 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"context"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/blob/dao"
"github.com/goharbor/harbor/src/pkg/blob/models"
"github.com/goharbor/harbor/src/pkg/q"
)
// Blob alias `models.Blob` to make it natural to use the Manager
type Blob = models.Blob
// ListParams alias `models.ListParams` to make it natural to use the Manager
type ListParams = models.ListParams
var (
// Mgr default blob manager
Mgr = NewManager()
)
// Manager interface provide the management functions for blobs
type Manager interface {
// AssociateWithArtifact associate blob with artifact
AssociateWithArtifact(ctx context.Context, blobDigest, artifactDigest string) (int64, error)
// AssociateWithProject associate blob with project
AssociateWithProject(ctx context.Context, blobID, projectID int64) (int64, error)
// Create create blob
Create(ctx context.Context, digest string, contentType string, size int64) (int64, error)
// CleanupAssociationsForArtifact remove all associations between blob and artifact by artifact digest
CleanupAssociationsForArtifact(ctx context.Context, artifactDigest string) error
// CleanupAssociationsForProject remove unneeded associations between blobs and project
CleanupAssociationsForProject(ctx context.Context, projectID int64, blobs []*Blob) error
// Get get blob by digest
Get(ctx context.Context, digest string) (*Blob, error)
// Update the blob
Update(ctx context.Context, blob *Blob) error
// List returns blobs by params
List(ctx context.Context, params ListParams) ([]*Blob, error)
// IsAssociatedWithArtifact returns true when blob associated with artifact
IsAssociatedWithArtifact(ctx context.Context, blobDigest, artifactDigest string) (bool, error)
// IsAssociatedWithProject returns true when blob associated with project
IsAssociatedWithProject(ctx context.Context, digest string, projectID int64) (bool, error)
}
type manager struct {
dao dao.DAO
}
func (m *manager) AssociateWithArtifact(ctx context.Context, blobDigest, artifactDigest string) (int64, error) {
return m.dao.CreateArtifactAndBlob(ctx, artifactDigest, blobDigest)
}
func (m *manager) AssociateWithProject(ctx context.Context, blobID, projectID int64) (int64, error) {
return m.dao.CreateProjectBlob(ctx, projectID, blobID)
}
func (m *manager) Create(ctx context.Context, digest string, contentType string, size int64) (int64, error) {
return m.dao.CreateBlob(ctx, &Blob{Digest: digest, ContentType: contentType, Size: size})
}
func (m *manager) CleanupAssociationsForArtifact(ctx context.Context, artifactDigest string) error {
return m.dao.DeleteArtifactAndBlobByArtifact(ctx, artifactDigest)
}
func (m *manager) CleanupAssociationsForProject(ctx context.Context, projectID int64, blobs []*Blob) error {
if len(blobs) == 0 {
return nil
}
shouldUnassociatedBlobs, err := m.dao.FindBlobsShouldUnassociatedWithProject(ctx, projectID, blobs)
if err != nil {
return err
}
var blobIDs []int64
for _, blob := range shouldUnassociatedBlobs {
blobIDs = append(blobIDs, blob.ID)
}
return m.dao.DeleteProjectBlob(ctx, projectID, blobIDs...)
}
func (m *manager) Get(ctx context.Context, digest string) (*Blob, error) {
return m.dao.GetBlobByDigest(ctx, digest)
}
func (m *manager) Update(ctx context.Context, blob *Blob) error {
return m.dao.UpdateBlob(ctx, blob)
}
func (m *manager) List(ctx context.Context, params ListParams) ([]*Blob, error) {
kw := q.KeyWords{}
if params.ArtifactDigest != "" {
blobDigests, err := m.dao.GetAssociatedBlobDigestsForArtifact(ctx, params.ArtifactDigest)
if err != nil {
return nil, err
}
params.BlobDigests = append(params.BlobDigests, blobDigests...)
}
if len(params.BlobDigests) > 0 {
kw["digest__in"] = params.BlobDigests
}
blobs, err := m.dao.ListBlobs(ctx, q.New(kw))
if err != nil {
return nil, err
}
var results []*Blob
for _, blob := range blobs {
results = append(results, blob)
}
return results, nil
}
func (m *manager) IsAssociatedWithArtifact(ctx context.Context, blobDigest, artifactDigest string) (bool, error) {
md, err := m.dao.GetArtifactAndBlob(ctx, artifactDigest, blobDigest)
if err != nil && !ierror.IsNotFoundErr(err) {
return false, err
}
return md != nil, nil
}
func (m *manager) IsAssociatedWithProject(ctx context.Context, digest string, projectID int64) (bool, error) {
return m.dao.ExistProjectBlob(ctx, projectID, digest)
}
// NewManager returns blob manager
func NewManager() Manager {
return &manager{dao: dao.New()}
}

253
src/pkg/blob/manager_test.go Normal file
View File

@ -0,0 +1,253 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"testing"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/stretchr/testify/suite"
)
type ManagerTestSuite struct {
htesting.Suite
}
func (suite *ManagerTestSuite) SetupSuite() {
suite.Suite.SetupSuite()
suite.Suite.ClearTables = []string{"artifact_blob", "project_blob", "blob"}
}
func (suite *ManagerTestSuite) TestAssociateWithArtifact() {
ctx := suite.Context()
artifactDigest := suite.DigestString()
blobDigest := suite.DigestString()
_, err := Mgr.AssociateWithArtifact(ctx, blobDigest, artifactDigest)
suite.Nil(err)
associated, err := Mgr.IsAssociatedWithArtifact(ctx, blobDigest, artifactDigest)
suite.Nil(err)
suite.True(associated)
}
func (suite *ManagerTestSuite) TestAssociateWithProject() {
ctx := suite.Context()
digest := suite.DigestString()
blobID, err := Mgr.Create(ctx, digest, "media type", 100)
suite.Nil(err)
projectID := int64(1)
_, err = Mgr.AssociateWithProject(ctx, blobID, projectID)
suite.Nil(err)
associated, err := Mgr.IsAssociatedWithProject(ctx, digest, projectID)
suite.Nil(err)
suite.True(associated)
}
func (suite *ManagerTestSuite) TestCleanupAssociationsForArtifact() {
ctx := suite.Context()
artifactDigest := suite.DigestString()
blob1Digest := suite.DigestString()
blob2Digest := suite.DigestString()
for _, digest := range []string{blob1Digest, blob2Digest} {
_, err := Mgr.AssociateWithArtifact(ctx, digest, artifactDigest)
suite.Nil(err)
associated, err := Mgr.IsAssociatedWithArtifact(ctx, digest, artifactDigest)
suite.Nil(err)
suite.True(associated)
}
suite.Nil(Mgr.CleanupAssociationsForArtifact(ctx, artifactDigest))
for _, digest := range []string{blob1Digest, blob2Digest} {
associated, err := Mgr.IsAssociatedWithArtifact(ctx, digest, artifactDigest)
suite.Nil(err)
suite.False(associated)
}
}
func (suite *ManagerTestSuite) TestCleanupAssociationsForProject() {
suite.WithProject(func(projectID int64, projectName string) {
artifact1 := suite.DigestString()
artifact2 := suite.DigestString()
sql := `INSERT INTO artifact_2 ("type", media_type, manifest_media_type, digest, project_id, repository_id) VALUES ('image', 'media_type', 'manifest_media_type', ?, ?, ?)`
suite.ExecSQL(sql, artifact1, projectID, 10)
suite.ExecSQL(sql, artifact2, projectID, 10)
defer suite.ExecSQL(`DELETE FROM artifact_2 WHERE project_id = ?`, projectID)
digest1 := suite.DigestString()
digest2 := suite.DigestString()
digest3 := suite.DigestString()
digest4 := suite.DigestString()
digest5 := suite.DigestString()
ctx := suite.Context()
blobDigests := []string{digest1, digest2, digest3, digest4, digest5}
for _, digest := range blobDigests {
blobID, err := Mgr.Create(ctx, digest, "media type", 100)
if suite.Nil(err) {
Mgr.AssociateWithProject(ctx, blobID, projectID)
}
}
blobs, err := Mgr.List(ctx, ListParams{BlobDigests: blobDigests})
suite.Nil(err)
suite.Len(blobs, 5)
for _, digest := range []string{digest1, digest2, digest3} {
Mgr.AssociateWithArtifact(ctx, digest, artifact1)
}
for _, digest := range blobDigests {
Mgr.AssociateWithArtifact(ctx, digest, artifact2)
}
{
suite.Nil(Mgr.CleanupAssociationsForProject(ctx, projectID, blobs))
for _, digest := range blobDigests {
associated, err := Mgr.IsAssociatedWithProject(ctx, digest, projectID)
suite.Nil(err)
suite.True(associated)
}
}
suite.ExecSQL(`DELETE FROM artifact_2 WHERE digest = ?`, artifact2)
{
suite.Nil(Mgr.CleanupAssociationsForProject(ctx, projectID, blobs))
for _, digest := range []string{digest1, digest2, digest3} {
associated, err := Mgr.IsAssociatedWithProject(ctx, digest, projectID)
suite.Nil(err)
suite.True(associated)
}
for _, digest := range []string{digest4, digest5} {
associated, err := Mgr.IsAssociatedWithProject(ctx, digest, projectID)
suite.Nil(err)
suite.False(associated)
}
}
})
}
func (suite *ManagerTestSuite) TestGet() {
ctx := suite.Context()
digest := suite.DigestString()
blob, err := Mgr.Get(ctx, digest)
suite.IsNotFoundErr(err)
suite.Nil(blob)
_, err = Mgr.Create(ctx, digest, "media type", 100)
suite.Nil(err)
blob, err = Mgr.Get(ctx, digest)
if suite.Nil(err) {
suite.Equal(digest, blob.Digest)
suite.Equal("media type", blob.ContentType)
suite.Equal(int64(100), blob.Size)
}
}
func (suite *ManagerTestSuite) TestUpdate() {
ctx := suite.Context()
digest := suite.DigestString()
_, err := Mgr.Create(ctx, digest, "media type", 100)
suite.Nil(err)
blob, err := Mgr.Get(ctx, digest)
if suite.Nil(err) {
blob.Size = 1000
suite.Nil(Mgr.Update(ctx, blob))
{
blob, err := Mgr.Get(ctx, digest)
suite.Nil(err)
suite.Equal(digest, blob.Digest)
suite.Equal("media type", blob.ContentType)
suite.Equal(int64(1000), blob.Size)
}
}
}
func (suite *ManagerTestSuite) TestList() {
ctx := suite.Context()
digest1 := suite.DigestString()
digest2 := suite.DigestString()
blobs, err := Mgr.List(ctx, ListParams{BlobDigests: []string{digest1, digest2}})
suite.Nil(err)
suite.Len(blobs, 0)
Mgr.Create(ctx, digest1, "media type", 100)
Mgr.Create(ctx, digest2, "media type", 100)
blobs, err = Mgr.List(ctx, ListParams{BlobDigests: []string{digest1, digest2}})
suite.Nil(err)
suite.Len(blobs, 2)
}
func (suite *ManagerTestSuite) TestListByArtifact() {
ctx := suite.Context()
artifact1 := suite.DigestString()
artifact2 := suite.DigestString()
digest1 := suite.DigestString()
digest2 := suite.DigestString()
digest3 := suite.DigestString()
digest4 := suite.DigestString()
digest5 := suite.DigestString()
blobDigests := []string{digest1, digest2, digest3, digest4, digest5}
for _, digest := range blobDigests {
Mgr.Create(ctx, digest, "media type", 100)
}
for i, digest := range blobDigests {
Mgr.AssociateWithArtifact(ctx, digest, artifact1)
if i < 3 {
Mgr.AssociateWithArtifact(ctx, digest, artifact2)
}
}
blobs, err := Mgr.List(ctx, ListParams{ArtifactDigest: artifact1})
suite.Nil(err)
suite.Len(blobs, 5)
blobs, err = Mgr.List(ctx, ListParams{ArtifactDigest: artifact2})
suite.Nil(err)
suite.Len(blobs, 3)
}
func TestManagerTestSuite(t *testing.T) {
suite.Run(t, &ManagerTestSuite{})
}

36
src/pkg/blob/models/blob.go Normal file
View File

@ -0,0 +1,36 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"github.com/goharbor/harbor/src/common/models"
)
// TODO: move ArtifactAndBlob, Blob and ProjectBlob to here
// ArtifactAndBlob alias ArtifactAndBlob model
type ArtifactAndBlob = models.ArtifactAndBlob
// Blob alias Blob model
type Blob = models.Blob
// ProjectBlob alias ProjectBlob model
type ProjectBlob = models.ProjectBlob
// ListParams list params
type ListParams struct {
ArtifactDigest string // list blobs which associated with the artifact
BlobDigests []string // list blobs which digest in the digests
}

97
src/pkg/distribution/distribution.go Normal file
View File

@ -0,0 +1,97 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package distribution
import (
"fmt"
"regexp"
"github.com/docker/distribution"
// docker schema1 manifest
_ "github.com/docker/distribution/manifest/schema1"
// docker schema2 manifest
_ "github.com/docker/distribution/manifest/schema2"
// manifestlist
_ "github.com/docker/distribution/manifest/manifestlist"
ref "github.com/docker/distribution/reference"
"github.com/goharbor/harbor/src/common/utils"
)
// Descriptor alias type of github.com/docker/distribution.Descriptor
type Descriptor = distribution.Descriptor
// Manifest alias type of github.com/docker/distribution.Manifest
type Manifest = distribution.Manifest
var (
// UnmarshalManifest alias func from `github.com/docker/distribution`
UnmarshalManifest = distribution.UnmarshalManifest
)
var (
name = fmt.Sprintf("(?P<name>%s)", ref.NameRegexp)
reference = fmt.Sprintf("(?P<reference>(%s|%s))", ref.TagRegexp, ref.DigestRegexp)
sessionID = "(?P<session_id>[a-zA-Z0-9-_.=]+)"
// BlobUploadURLRegexp regexp which match blob upload url
BlobUploadURLRegexp = regexp.MustCompile(`^/v2/` + name + `/blobs/uploads/` + sessionID)
// InitiateBlobUploadRegexp regexp which match initiate blob upload url
InitiateBlobUploadRegexp = regexp.MustCompile(`^/v2/` + name + `/blobs/uploads`)
// ManifestURLRegexp regexp which match manifest url
ManifestURLRegexp = regexp.MustCompile(`^/v2/` + name + `/manifests/` + reference)
)
var (
extractNameRegexp = regexp.MustCompile(`^/v2/` + name + `/(manifests|blobs|tags)`)
extractSessionIDRegexp = regexp.MustCompile(`^/v2/` + name + `/blobs/uploads/` + sessionID)
)
// ParseName returns name value from distribution API URL path
func ParseName(path string) string {
m := findNamedMatches(extractNameRegexp, path)
if len(m) > 0 {
return m["name"]
}
return ""
}
// ParseProjectName returns project name from distribution API URL path
func ParseProjectName(path string) string {
projectName, _ := utils.ParseRepository(ParseName(path))
return projectName
}
// ParseSessionID returns session id value from distribution API URL path
func ParseSessionID(path string) string {
m := findNamedMatches(extractSessionIDRegexp, path)
if len(m) > 0 {
return m["session_id"]
}
return ""
}
func findNamedMatches(regex *regexp.Regexp, str string) map[string]string {
match := regex.FindStringSubmatch(str)
results := map[string]string{}
for i, name := range match {
results[regex.SubexpNames()[i]] = name
}
return results
}

98
src/pkg/distribution/distribution_test.go Normal file
View File

@ -0,0 +1,98 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package distribution
import (
"testing"
_ "github.com/docker/distribution/manifest/manifestlist"
_ "github.com/docker/distribution/manifest/schema1"
_ "github.com/docker/distribution/manifest/schema2"
)
func TestParseSessionID(t *testing.T) {
type args struct {
path string
}
tests := []struct {
name string
args args
want string
}{
{"base", args{"/v2"}, ""},
{"tags", args{"/v2/library/photon/tags/list"}, ""},
{"manifest", args{"/v2/library/photon/manifests/2.0"}, ""},
{"blob", args{"/v2/library/photon/blobs/sha256:c52fca2e807cb7807cfd831d6df45a332d5826a97f886f7da0e9c61842f9ce1e"}, ""},
{"initiate blob upload", args{"/v2/library/photon/blobs/uploads"}, ""},
{"blob upload", args{"/v2/library/photon/blobs/uploads/aa41e8cb-21b4-423c-b533-9e4b084075c7"}, "aa41e8cb-21b4-423c-b533-9e4b084075c7"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := ParseSessionID(tt.args.path); got != tt.want {
t.Errorf("ParseSessionID() = %v, want %v", got, tt.want)
}
})
}
}
func TestParseName(t *testing.T) {
type args struct {
path string
}
tests := []struct {
name string
args args
want string
}{
{"base", args{"/v2"}, ""},
{"tags", args{"/v2/library/photon/tags/list"}, "library/photon"},
{"manifest", args{"/v2/library/photon/manifests/2.0"}, "library/photon"},
{"blob", args{"/v2/library/photon/blobs/sha256:c52fca2e807cb7807cfd831d6df45a332d5826a97f886f7da0e9c61842f9ce1e"}, "library/photon"},
{"initiate blob upload", args{"/v2/library/photon/blobs/uploads"}, "library/photon"},
{"blob upload", args{"/v2/library/photon/blobs/uploads/aa41e8cb-21b4-423c-b533-9e4b084075c7"}, "library/photon"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := ParseName(tt.args.path); got != tt.want {
t.Errorf("ParseName() = %v, want %v", got, tt.want)
}
})
}
}
func TestParseProjectName(t *testing.T) {
type args struct {
path string
}
tests := []struct {
name string
args args
want string
}{
{"base", args{"/v2"}, ""},
{"tags", args{"/v2/library/photon/tags/list"}, "library"},
{"manifest", args{"/v2/library/photon/manifests/2.0"}, "library"},
{"blob", args{"/v2/library/photon/blobs/sha256:c52fca2e807cb7807cfd831d6df45a332d5826a97f886f7da0e9c61842f9ce1e"}, "library"},
{"initiate blob upload", args{"/v2/library/photon/blobs/uploads"}, "library"},
{"blob upload", args{"/v2/library/photon/blobs/uploads/aa41e8cb-21b4-423c-b533-9e4b084075c7"}, "library"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := ParseProjectName(tt.args.path); got != tt.want {
t.Errorf("ParseProjectName() = %v, want %v", got, tt.want)
}
})
}
}

10
src/pkg/q/query.go
View File

@ -14,6 +14,9 @@
package q
// KeyWords ...
type KeyWords = map[string]interface{}
// Query parameters
type Query struct {
// Page number
@ -21,7 +24,12 @@ type Query struct {
// Page size
PageSize int64
// List of key words
Keywords map[string]interface{}
Keywords KeyWords
}
// New returns Query with keywords
func New(kw KeyWords) *Query {
return &Query{Keywords: kw}
}
// Copy the specified query object

25
src/server/middleware/blob/controller.go Normal file
View File

@ -0,0 +1,25 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"github.com/goharbor/harbor/src/api/blob"
"github.com/goharbor/harbor/src/api/project"
)
var (
blobController = blob.Ctl
projectController = project.Ctl
)

70
src/server/middleware/blob/patch_blob_upload.go Normal file
View File

@ -0,0 +1,70 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"fmt"
"net/http"
"strconv"
"strings"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/server/middleware"
)
// PatchBlobUploadMiddleware middleware to record the accepted blob size for stream blob upload
func PatchBlobUploadMiddleware() func(http.Handler) http.Handler {
return middleware.AfterResponse(func(w http.ResponseWriter, r *http.Request, statusCode int) error {
// Only record when patch blob upload success
if statusCode != http.StatusAccepted {
return nil
}
size, err := parseAcceptedBlobSize(w.Header().Get("Range"))
if err != nil {
return err
}
sessionID := distribution.ParseSessionID(r.URL.Path)
return blobController.SetAcceptedBlobSize(sessionID, size)
})
}
// parseAcceptedBlobSize parse the blob stream upload response and return the size blob accepted
func parseAcceptedBlobSize(rangeHeader string) (int64, error) {
// Range: Range indicating the current progress of the upload.
// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#get-blob-upload
if rangeHeader == "" {
return 0, fmt.Errorf("range header required")
}
parts := strings.SplitN(rangeHeader, "-", 2)
if len(parts) != 2 {
return 0, fmt.Errorf("range header bad value: %s", rangeHeader)
}
size, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
return 0, err
}
// docker registry did '-1' in the response
if size > 0 {
size = size + 1
}
return size, nil
}

60
src/server/middleware/blob/patch_blob_upload_test.go Normal file
View File

@ -0,0 +1,60 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/goharbor/harbor/src/api/blob"
"github.com/google/uuid"
"github.com/stretchr/testify/suite"
)
type PatchBlobUploadMiddlewareTestSuite struct {
suite.Suite
}
func (suite *PatchBlobUploadMiddlewareTestSuite) TestMiddleware() {
next := func(rangeHeader string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusAccepted)
w.Header().Set("Range", rangeHeader)
})
}
sessionID := uuid.New().String()
path := fmt.Sprintf("/v2/library/photon/blobs/uploads/%s", sessionID)
req := httptest.NewRequest(http.MethodPatch, path, nil)
res := httptest.NewRecorder()
PatchBlobUploadMiddleware()(next("bad value")).ServeHTTP(res, req)
suite.Equal(http.StatusInternalServerError, res.Code)
req = httptest.NewRequest(http.MethodPatch, path, nil)
res = httptest.NewRecorder()
PatchBlobUploadMiddleware()(next("0-511")).ServeHTTP(res, req)
suite.Equal(http.StatusAccepted, res.Code)
size, err := blob.Ctl.GetAcceptedBlobSize(sessionID)
suite.Nil(err)
suite.Equal(int64(512), size)
}
func TestPatchBlobUploadMiddlewareTestSuite(t *testing.T) {
suite.Run(t, &PatchBlobUploadMiddlewareTestSuite{})
}

56
src/server/middleware/blob/post_initiate_blob_upload.go Normal file
View File

@ -0,0 +1,56 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"fmt"
"net/http"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/server/middleware"
)
// PostInitiateBlobUploadMiddleware middleware to add blob to project after mount blob success
func PostInitiateBlobUploadMiddleware() func(http.Handler) http.Handler {
return middleware.AfterResponse(func(w http.ResponseWriter, r *http.Request, statusCode int) error {
if statusCode != http.StatusCreated {
return nil
}
query := r.URL.Query()
mount := query.Get("mount")
if mount == "" {
return nil
}
logPrefix := fmt.Sprintf("[middleware][%s][blob]", r.URL.Path)
ctx := r.Context()
project, err := projectController.GetByName(ctx, distribution.ParseProjectName(r.URL.Path))
if err != nil {
log.Errorf("%s: get project failed, error: %v", logPrefix, err)
return err
}
if err := blobController.AssociateWithProjectByDigest(ctx, mount, project.ProjectID); err != nil {
log.Errorf("%s: mount blob %s to project %s failed, error: %v", logPrefix, mount, project.Name, err)
return err
}
return nil
})
}

69
src/server/middleware/blob/post_initiate_blob_upload_test.go Normal file
View File

@ -0,0 +1,69 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/goharbor/harbor/src/api/blob"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/stretchr/testify/suite"
)
type PostInitiateBlobUploadMiddlewareTestSuite struct {
htesting.Suite
}
func (suite *PostInitiateBlobUploadMiddlewareTestSuite) SetupSuite() {
suite.Suite.SetupSuite()
suite.Suite.ClearTables = []string{"project_blob", "blob"}
}
func (suite *PostInitiateBlobUploadMiddlewareTestSuite) TestMountBlob() {
suite.WithProject(func(projectID int64, projectName string) {
ctx := suite.Context()
digest := suite.DigestString()
_, err := blob.Ctl.Ensure(ctx, digest, "", 512)
suite.Nil(err)
suite.WithProject(func(id int64, name string) {
query := map[string]string{"mount": digest}
req := suite.NewRequest(http.MethodPost, fmt.Sprintf("/v2/%s/photon/blobs/uploads", name), nil, query)
res := httptest.NewRecorder()
next := suite.NextHandler(http.StatusCreated, nil)
PostInitiateBlobUploadMiddleware()(next).ServeHTTP(res, req)
exist, err := blob.Ctl.Exist(ctx, digest, blob.IsAssociatedWithProject(id))
suite.Nil(err)
suite.True(exist)
blob, err := blob.Ctl.Get(ctx, digest)
if suite.Nil(err) {
suite.Equal(digest, blob.Digest)
suite.Equal(int64(512), blob.Size)
}
})
})
}
func TestPostInitiateBlobUploadMiddlewareTestSuite(t *testing.T) {
suite.Run(t, &PostInitiateBlobUploadMiddlewareTestSuite{})
}

67
src/server/middleware/blob/put_blob_upload.go Normal file
View File

@ -0,0 +1,67 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"fmt"
"net/http"
"strconv"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/server/middleware"
)
// PutBlobUploadMiddleware middleware to create Blob and ProjectBlob after PUT /v2/<name>/blobs/uploads/<session_id> success
func PutBlobUploadMiddleware() func(http.Handler) http.Handler {
return middleware.AfterResponse(func(w http.ResponseWriter, r *http.Request, statusCode int) error {
if statusCode != http.StatusCreated {
return nil
}
logPrefix := fmt.Sprintf("[middleware][%s][blob]", r.URL.Path)
size, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64)
if err != nil || size == 0 {
size, err = blobController.GetAcceptedBlobSize(distribution.ParseSessionID(r.URL.Path))
}
if err != nil {
log.Errorf("%s: get blob size failed, error: %v", logPrefix, err)
return err
}
ctx := r.Context()
p, err := projectController.GetByName(ctx, distribution.ParseProjectName(r.URL.Path))
if err != nil {
log.Errorf("%s: get project failed, error: %v", logPrefix, err)
return err
}
digest := w.Header().Get("Docker-Content-Digest")
blobID, err := blobController.Ensure(ctx, digest, "application/octet-stream", size)
if err != nil {
log.Errorf("%s: ensure blob %s failed, error: %v", logPrefix, digest, err)
return err
}
if err := blobController.AssociateWithProjectByID(ctx, blobID, p.ProjectID); err != nil {
log.Errorf("%s: associate blob %s with project %s failed, error: %v", logPrefix, digest, p.Name, err)
return err
}
return nil
})
}

97
src/server/middleware/blob/put_blob_upload_test.go Normal file
View File

@ -0,0 +1,97 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/goharbor/harbor/src/api/blob"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/google/uuid"
"github.com/stretchr/testify/suite"
)
type PutBlobUploadMiddlewareTestSuite struct {
htesting.Suite
}
func (suite *PutBlobUploadMiddlewareTestSuite) SetupSuite() {
suite.Suite.SetupSuite()
suite.Suite.ClearTables = []string{"project_blob", "blob"}
}
func (suite *PutBlobUploadMiddlewareTestSuite) TestDataInBody() {
suite.WithProject(func(projectID int64, projectName string) {
req := suite.NewRequest(http.MethodPut, fmt.Sprintf("/v2/%s/photon/blobs/uploads/%s", projectName, uuid.New().String()), nil)
req.Header.Set("Content-Length", "512")
res := httptest.NewRecorder()
digest := suite.DigestString()
next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": digest})
PutBlobUploadMiddleware()(next).ServeHTTP(res, req)
exist, err := blob.Ctl.Exist(suite.Context(), digest, blob.IsAssociatedWithProject(projectID))
suite.Nil(err)
suite.True(exist)
blob, err := blob.Ctl.Get(suite.Context(), digest)
suite.Nil(err)
suite.Equal(digest, blob.Digest)
suite.Equal(int64(512), blob.Size)
})
}
func (suite *PutBlobUploadMiddlewareTestSuite) TestWithoutBody() {
suite.WithProject(func(projectID int64, projectName string) {
sessionID := uuid.New().String()
path := fmt.Sprintf("/v2/%s/photon/blobs/uploads/%s", projectName, sessionID)
{
req := httptest.NewRequest(http.MethodPatch, path, nil)
res := httptest.NewRecorder()
next := suite.NextHandler(http.StatusAccepted, map[string]string{"Range": "0-511"})
PatchBlobUploadMiddleware()(next).ServeHTTP(res, req)
suite.Equal(http.StatusAccepted, res.Code)
}
req := suite.NewRequest(http.MethodPut, path, nil)
res := httptest.NewRecorder()
digest := suite.DigestString()
next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": digest})
PutBlobUploadMiddleware()(next).ServeHTTP(res, req)
suite.Equal(http.StatusCreated, res.Code)
exist, err := blob.Ctl.Exist(suite.Context(), digest, blob.IsAssociatedWithProject(projectID))
suite.Nil(err)
suite.True(exist)
blob, err := blob.Ctl.Get(suite.Context(), digest)
if suite.Nil(err) {
suite.Equal(digest, blob.Digest)
suite.Equal(int64(512), blob.Size)
}
})
}
func TestPutBlobUploadMiddlewareTestSuite(t *testing.T) {
suite.Run(t, &PutBlobUploadMiddlewareTestSuite{})
}

121
src/server/middleware/blob/put_manifest.go Normal file
View File

@ -0,0 +1,121 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"fmt"
"io/ioutil"
"net/http"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/distribution"
"github.com/goharbor/harbor/src/server/middleware"
"github.com/justinas/alice"
)
// PutManifestMiddleware middleware which create Blobs for the foreign layers and associate them with the project,
// update the content type of the Blobs which already exist,
// create Blob for the manifest, associate all Blobs with the manifest after PUT /v2/<name>/manifests/<reference> success.
func PutManifestMiddleware() func(http.Handler) http.Handler {
before := middleware.BeforeRequest(func(r *http.Request) error {
// Do nothing, only make the request nopclose
return nil
})
after := middleware.AfterResponse(func(w http.ResponseWriter, r *http.Request, statusCode int) error {
if statusCode != http.StatusCreated {
return nil
}
logPrefix := fmt.Sprintf("[middleware][%s][blob]", r.URL.Path)
ctx := r.Context()
p, err := projectController.GetByName(ctx, distribution.ParseProjectName(r.URL.Path))
if err != nil {
log.Errorf("%s: get project failed, error: %v", logPrefix, err)
return err
}
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return err
}
contentType := r.Header.Get("Content-Type")
manifest, descriptor, err := distribution.UnmarshalManifest(contentType, body)
if err != nil {
log.Errorf("%s: unmarshal manifest failed, error: %v", logPrefix, err)
return err
}
// sync blobs
if err := blobController.Sync(ctx, manifest.References()); err != nil {
log.Errorf("%s: sync missing blobs from manifest %s failed, error: %c", logPrefix, descriptor.Digest.String(), err)
return err
}
for _, digest := range findForeignBlobDigests(manifest) {
if err := blobController.AssociateWithProjectByDigest(ctx, digest, p.ProjectID); err != nil {
return err
}
}
artifactDigest := descriptor.Digest.String()
// ensure Blob for the manifest
blobID, err := blobController.Ensure(ctx, artifactDigest, contentType, descriptor.Size)
if err != nil {
log.Errorf("%s: ensure blob %s failed, error: %v", logPrefix, descriptor.Digest, err)
return err
}
if err := blobController.AssociateWithProjectByID(ctx, blobID, p.ProjectID); err != nil {
log.Errorf("%s: associate manifest with artifact %s failed, error: %v", logPrefix, descriptor.Digest, err)
return err
}
var blobDigests []string
for _, reference := range manifest.References() {
blobDigests = append(blobDigests, reference.Digest.String())
}
// associate blobs of the manifest with artifact
if err := blobController.AssociateWithArtifact(ctx, blobDigests, artifactDigest); err != nil {
log.Errorf("%s: associate blobs with artifact %s failed, error: %v", logPrefix, descriptor.Digest, err)
return err
}
return nil
})
return func(next http.Handler) http.Handler {
return alice.New(before, after).Then(next)
}
}
func isForeign(d *distribution.Descriptor) bool {
return d.MediaType == schema2.MediaTypeForeignLayer
}
func findForeignBlobDigests(manifest distribution.Manifest) []string {
var digests []string
for _, reference := range manifest.References() {
if isForeign(&reference) {
digests = append(digests, reference.Digest.String())
}
}
return digests
}

143
src/server/middleware/blob/put_manifest_test.go Normal file
View File

@ -0,0 +1,143 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blob
import (
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/goharbor/harbor/src/api/blob"
"github.com/goharbor/harbor/src/pkg/distribution"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/google/uuid"
"github.com/stretchr/testify/suite"
)
type PutManifestMiddlewareTestSuite struct {
htesting.Suite
}
func (suite *PutManifestMiddlewareTestSuite) SetupSuite() {
suite.Suite.SetupSuite()
suite.Suite.ClearTables = []string{"project_blob", "blob", "artifact_blob"}
}
func (suite *PutManifestMiddlewareTestSuite) pushBlob(name string, digest string, size int64) {
req := suite.NewRequest(http.MethodPut, fmt.Sprintf("/v2/%s/blobs/uploads/%s", name, uuid.New().String()), nil)
req.Header.Set("Content-Length", fmt.Sprintf("%d", size))
res := httptest.NewRecorder()
next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": digest})
PutBlobUploadMiddleware()(next).ServeHTTP(res, req)
suite.Equal(res.Code, http.StatusCreated)
}
func (suite *PutManifestMiddlewareTestSuite) TestMiddleware() {
body := `
{
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"config": {
"mediaType": "application/vnd.docker.container.image.v1+json",
"size": 6868,
"digest": "sha256:9b188f5fb1e6e1c7b10045585cb386892b2b4e1d31d62e3688c6fa8bf9fd32b5"
},
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
"size": 27092274,
"digest": "sha256:8ec398bc03560e0fa56440e96da307cdf0b1ad153f459b52bca53ae7ddb8236d"
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": 1730,
"digest": "sha256:da01136793fac089b2ff13c2bf3c9d5d5550420fbd9981e08198fd251a0ab7b4"
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": 1357602,
"digest": "sha256:cf1486a2c0b86ddb45238e86c6bf9666c20113f7878e4cd4fa175fd74ac5d5b7"
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": 7344202,
"digest": "sha256:a44f7da98d9e65b723ee913a9e6758db120a43fcce564b3dcf61cb9eb2823dad"
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": 97,
"digest": "sha256:c677fde73875fc4c1e38ccdc791fe06380be0468fac220358f38c910e336266e"
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": 409,
"digest": "sha256:727f8da63ac248054cb7dda635ee16da76e553ec99be565a54180c83d04025a8"
}
]
}`
manifest, descriptor, err := distribution.UnmarshalManifest("application/vnd.docker.distribution.manifest.v2+json", []byte(body))
suite.Nil(err)
suite.WithProject(func(projectID int64, projectName string) {
name := fmt.Sprintf("%s/redis", projectName)
for _, reference := range manifest.References() {
if !isForeign(&reference) {
suite.pushBlob(name, reference.Digest.String(), reference.Size)
}
}
req := suite.NewRequest(http.MethodPut, fmt.Sprintf("/v2/%s/manifests/%s", name, descriptor.Digest.String()), strings.NewReader(body))
req.Header.Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json")
res := httptest.NewRecorder()
next := suite.NextHandler(http.StatusCreated, map[string]string{"Docker-Content-Digest": descriptor.Digest.String()})
PutManifestMiddleware()(next).ServeHTTP(res, req)
suite.Equal(http.StatusCreated, res.Code)
for _, reference := range manifest.References() {
opts := []blob.Option{
blob.IsAssociatedWithArtifact(descriptor.Digest.String()),
blob.IsAssociatedWithProject(projectID),
}
b, err := blob.Ctl.Get(suite.Context(), reference.Digest.String(), opts...)
if suite.Nil(err) {
suite.Equal(reference.MediaType, b.ContentType)
suite.Equal(reference.Size, b.Size)
}
}
{
opts := []blob.Option{
blob.IsAssociatedWithArtifact(descriptor.Digest.String()),
blob.IsAssociatedWithProject(projectID),
}
b, err := blob.Ctl.Get(suite.Context(), descriptor.Digest.String(), opts...)
if suite.Nil(err) {
suite.Equal(descriptor.MediaType, b.ContentType)
suite.Equal(descriptor.Size, b.Size)
}
}
})
}
func TestPutManifestMiddlewareTestSuite(t *testing.T) {
suite.Run(t, &PutManifestMiddlewareTestSuite{})
}

34
src/server/middleware/middleware.go
View File

@ -16,6 +16,9 @@ package middleware
import (
"net/http"
"github.com/goharbor/harbor/src/internal"
serror "github.com/goharbor/harbor/src/server/error"
)
// Middleware receives a handler and returns another handler.
@ -47,3 +50,34 @@ func New(fn func(http.ResponseWriter, *http.Request, http.Handler), skippers ...
})
}
}
// BeforeRequest make a middleware which will call hook before the next handler
func BeforeRequest(hook func(*http.Request) error, skippers ...Skipper) func(http.Handler) http.Handler {
return New(func(w http.ResponseWriter, r *http.Request, next http.Handler) {
if err := hook(internal.NopCloseRequest(r)); err != nil {
serror.SendError(w, err)
return
}
next.ServeHTTP(w, r)
}, skippers...)
}
// AfterResponse make a middleware which will call hook after the next handler
func AfterResponse(hook func(http.ResponseWriter, *http.Request, int) error, skippers ...Skipper) func(http.Handler) http.Handler {
return New(func(w http.ResponseWriter, r *http.Request, next http.Handler) {
res, ok := w.(*internal.ResponseBuffer)
if !ok {
res = internal.NewResponseBuffer(w)
defer res.Flush()
}
next.ServeHTTP(res, r)
if err := hook(res, r, res.StatusCode()); err != nil {
res.Reset()
serror.SendError(res, err)
}
}, skippers...)
}

22
src/server/registry/route.go
View File

@ -15,7 +15,10 @@
package registry
import (
"net/http"
"github.com/goharbor/harbor/src/server/middleware/artifactinfo"
"github.com/goharbor/harbor/src/server/middleware/blob"
"github.com/goharbor/harbor/src/server/middleware/contenttrust"
"github.com/goharbor/harbor/src/server/middleware/immutable"
"github.com/goharbor/harbor/src/server/middleware/manifestinfo"
@ -24,7 +27,6 @@ import (
"github.com/goharbor/harbor/src/server/middleware/v2auth"
"github.com/goharbor/harbor/src/server/middleware/vulnerable"
"github.com/goharbor/harbor/src/server/router"
"net/http"
)
// RegisterRoutes for OCI registry APIs
@ -69,7 +71,25 @@ func RegisterRoutes() {
Middleware(readonly.Middleware()).
Middleware(manifestinfo.Middleware()).
Middleware(immutable.MiddlewarePush()).
Middleware(blob.PutManifestMiddleware()).
HandlerFunc(putManifest)
// initiate blob upload
root.NewRoute().
Method(http.MethodPost).
Path("/*/blobs/uploads").
Middleware(blob.PostInitiateBlobUploadMiddleware()).
Handler(proxy)
// blob upload
root.NewRoute().
Method(http.MethodPatch).
Path("/*/blobs/uploads/:session_id").
Middleware(blob.PatchBlobUploadMiddleware()).
Handler(proxy)
root.NewRoute().
Method(http.MethodPut).
Path("/*/blobs/uploads/:session_id").
Middleware(blob.PutBlobUploadMiddleware()).
Handler(proxy)
// blob
root.NewRoute().
Method(http.MethodPost).

211
src/testing/pkg/blob/manager.go Normal file
View File

@ -0,0 +1,211 @@
// Code generated by mockery v1.0.0. DO NOT EDIT.
package blob
import (
context "context"
blobmodels "github.com/goharbor/harbor/src/pkg/blob/models"
mock "github.com/stretchr/testify/mock"
models "github.com/goharbor/harbor/src/common/models"
)
// Manager is an autogenerated mock type for the Manager type
type Manager struct {
mock.Mock
}
// AssociateWithArtifact provides a mock function with given fields: ctx, blobDigest, artifactDigest
func (_m *Manager) AssociateWithArtifact(ctx context.Context, blobDigest string, artifactDigest string) (int64, error) {
ret := _m.Called(ctx, blobDigest, artifactDigest)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, string, string) int64); ok {
r0 = rf(ctx, blobDigest, artifactDigest)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok {
r1 = rf(ctx, blobDigest, artifactDigest)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// AssociateWithProject provides a mock function with given fields: ctx, blobID, projectID
func (_m *Manager) AssociateWithProject(ctx context.Context, blobID int64, projectID int64) (int64, error) {
ret := _m.Called(ctx, blobID, projectID)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, int64, int64) int64); ok {
r0 = rf(ctx, blobID, projectID)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int64, int64) error); ok {
r1 = rf(ctx, blobID, projectID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// CleanupAssociationsForArtifact provides a mock function with given fields: ctx, artifactDigest
func (_m *Manager) CleanupAssociationsForArtifact(ctx context.Context, artifactDigest string) error {
ret := _m.Called(ctx, artifactDigest)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, string) error); ok {
r0 = rf(ctx, artifactDigest)
} else {
r0 = ret.Error(0)
}
return r0
}
// CleanupAssociationsForProject provides a mock function with given fields: ctx, projectID, blobs
func (_m *Manager) CleanupAssociationsForProject(ctx context.Context, projectID int64, blobs []*models.Blob) error {
ret := _m.Called(ctx, projectID, blobs)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, int64, []*models.Blob) error); ok {
r0 = rf(ctx, projectID, blobs)
} else {
r0 = ret.Error(0)
}
return r0
}
// Create provides a mock function with given fields: ctx, digest, contentType, size
func (_m *Manager) Create(ctx context.Context, digest string, contentType string, size int64) (int64, error) {
ret := _m.Called(ctx, digest, contentType, size)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, string, string, int64) int64); ok {
r0 = rf(ctx, digest, contentType, size)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string, string, int64) error); ok {
r1 = rf(ctx, digest, contentType, size)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Get provides a mock function with given fields: ctx, digest
func (_m *Manager) Get(ctx context.Context, digest string) (*models.Blob, error) {
ret := _m.Called(ctx, digest)
var r0 *models.Blob
if rf, ok := ret.Get(0).(func(context.Context, string) *models.Blob); ok {
r0 = rf(ctx, digest)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*models.Blob)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, digest)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// IsAssociatedWithArtifact provides a mock function with given fields: ctx, blobDigest, artifactDigest
func (_m *Manager) IsAssociatedWithArtifact(ctx context.Context, blobDigest string, artifactDigest string) (bool, error) {
ret := _m.Called(ctx, blobDigest, artifactDigest)
var r0 bool
if rf, ok := ret.Get(0).(func(context.Context, string, string) bool); ok {
r0 = rf(ctx, blobDigest, artifactDigest)
} else {
r0 = ret.Get(0).(bool)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok {
r1 = rf(ctx, blobDigest, artifactDigest)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// IsAssociatedWithProject provides a mock function with given fields: ctx, digest, projectID
func (_m *Manager) IsAssociatedWithProject(ctx context.Context, digest string, projectID int64) (bool, error) {
ret := _m.Called(ctx, digest, projectID)
var r0 bool
if rf, ok := ret.Get(0).(func(context.Context, string, int64) bool); ok {
r0 = rf(ctx, digest, projectID)
} else {
r0 = ret.Get(0).(bool)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string, int64) error); ok {
r1 = rf(ctx, digest, projectID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// List provides a mock function with given fields: ctx, params
func (_m *Manager) List(ctx context.Context, params blobmodels.ListParams) ([]*models.Blob, error) {
ret := _m.Called(ctx, params)
var r0 []*models.Blob
if rf, ok := ret.Get(0).(func(context.Context, blobmodels.ListParams) []*models.Blob); ok {
r0 = rf(ctx, params)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*models.Blob)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, blobmodels.ListParams) error); ok {
r1 = rf(ctx, params)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Update provides a mock function with given fields: ctx, _a1
func (_m *Manager) Update(ctx context.Context, _a1 *models.Blob) error {
ret := _m.Called(ctx, _a1)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, *models.Blob) error); ok {
r0 = rf(ctx, _a1)
} else {
r0 = ret.Error(0)
}
return r0
}

17
src/testing/pkg/pkg.go Normal file
View File

@ -0,0 +1,17 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pkg
//go:generate mockery -case snake -dir ../../pkg/blob -name Manager -output ./blob -outpkg blob

62
src/testing/pkg/project/manager.go
View File

@ -19,27 +19,59 @@ import (
"github.com/stretchr/testify/mock"
)
// FakeManager is a fake project manager that implement src/pkg/project.Manager interface
// FakeManager is an autogenerated mock type for the FakeManager type
type FakeManager struct {
mock.Mock
}
// List ...
func (f *FakeManager) List(query ...*models.ProjectQueryParam) ([]*models.Project, error) {
args := f.Called()
var projects []*models.Project
if args.Get(0) != nil {
projects = args.Get(0).([]*models.Project)
// Get provides a mock function with given fields: _a0
func (_m *FakeManager) Get(_a0 interface{}) (*models.Project, error) {
ret := _m.Called(_a0)
var r0 *models.Project
if rf, ok := ret.Get(0).(func(interface{}) *models.Project); ok {
r0 = rf(_a0)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*models.Project)
}
return projects, args.Error(1)
}
var r1 error
if rf, ok := ret.Get(1).(func(interface{}) error); ok {
r1 = rf(_a0)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Get ...
func (f *FakeManager) Get(interface{}) (*models.Project, error) {
args := f.Called()
var project *models.Project
if args.Get(0) != nil {
project = args.Get(0).(*models.Project)
// List provides a mock function with given fields: _a0
func (_m *FakeManager) List(_a0 ...*models.ProjectQueryParam) ([]*models.Project, error) {
_va := make([]interface{}, len(_a0))
for _i := range _a0 {
_va[_i] = _a0[_i]
}
return project, args.Error(1)
var _ca []interface{}
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
var r0 []*models.Project
if rf, ok := ret.Get(0).(func(...*models.ProjectQueryParam) []*models.Project); ok {
r0 = rf(_a0...)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*models.Project)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(...*models.ProjectQueryParam) error); ok {
r1 = rf(_a0...)
} else {
r1 = ret.Error(1)
}
return r0, r1
}

76
src/testing/suite.go
View File

@ -15,15 +15,24 @@
package testing
import (
"context"
"fmt"
"io"
"math/rand"
"net/http"
"net/http/httptest"
"strconv"
"sync"
"time"
o "github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/core/config"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/internal/orm"
"github.com/goharbor/harbor/src/pkg/types"
"github.com/opencontainers/go-digest"
"github.com/stretchr/testify/suite"
)
@ -31,15 +40,29 @@ func init() {
rand.Seed(time.Now().UnixNano())
}
var (
once sync.Once
)
// Suite ...
type Suite struct {
suite.Suite
ClearTables []string
}
// SetupSuite ...
func (suite *Suite) SetupSuite() {
once.Do(func() {
config.Init()
dao.PrepareTestForPostgresSQL()
})
}
// TearDownSuite ...
func (suite *Suite) TearDownSuite() {
for _, table := range suite.ClearTables {
dao.ClearTable(table)
}
}
// RandString ...
@ -57,6 +80,16 @@ func (suite *Suite) RandString(n int, letters ...string) string {
return string(b)
}
// Digest ...
func (suite *Suite) Digest() digest.Digest {
return digest.FromString(suite.RandString(128))
}
// DigestString ...
func (suite *Suite) DigestString() string {
return suite.Digest().String()
}
// WithProject ...
func (suite *Suite) WithProject(f func(int64, string), projectNames ...string) {
var projectName string
@ -81,6 +114,49 @@ func (suite *Suite) WithProject(f func(int64, string), projectNames ...string) {
f(projectID, projectName)
}
// Context ...
func (suite *Suite) Context() context.Context {
return orm.NewContext(context.TODO(), o.NewOrm())
}
// NewRequest ...
func (suite *Suite) NewRequest(method, target string, body io.Reader, queries ...map[string]string) *http.Request {
req := httptest.NewRequest(method, target, body)
if len(queries) > 0 {
q := req.URL.Query()
for key, value := range queries[0] {
q.Add(key, value)
}
req.URL.RawQuery = q.Encode()
}
return req.WithContext(suite.Context())
}
// NextHandler ...
func (suite *Suite) NextHandler(statusCode int, headers map[string]string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(statusCode)
for key, value := range headers {
w.Header().Set(key, value)
}
})
}
// ExecSQL ...
func (suite *Suite) ExecSQL(query string, args ...interface{}) {
o := o.NewOrm()
_, err := o.Raw(query, args...).Exec()
suite.Nil(err)
}
// IsNotFoundErr ...
func (suite *Suite) IsNotFoundErr(err error) bool {
return suite.True(ierror.IsNotFoundErr(err))
}
// AssertResourceUsage ...
func (suite *Suite) AssertResourceUsage(expected int64, resource types.ResourceName, projectID int64) {
usage := models.QuotaUsage{Reference: "project", ReferenceID: strconv.FormatInt(projectID, 10)}

2
tests/apitests/python/testutils.py
View File

@ -14,7 +14,7 @@ harbor_server = os.environ["HARBOR_HOST"]
#CLIENT=dict(endpoint="https://"+harbor_server+"/api")
ADMIN_CLIENT=dict(endpoint = os.environ.get("HARBOR_HOST_SCHEMA", "https")+ "://"+harbor_server+"/api/v2.0", username = admin_user, password = admin_pwd)
USER_ROLE=dict(admin=0,normal=1)
TEARDOWN = True
TEARDOWN = os.environ.get('TEARDOWN', 'true').lower() in ('true', 'yes')
def GetProductApi(username, password, harbor_server= os.environ["HARBOR_HOST"]):