Merge pull request #10733 from ywk253100/200213_copy_artifact

Implement copy artifact API

api/v2.0/swagger.yaml

@@ -187,6 +187,36 @@ paths:
           $ref: '#/responses/404'
         '500':
           $ref: '#/responses/500'
+    post:
+      summary: Copy artifact
+      description: Copy the artifact specified in the "from" parameter to the repository.
+      tags:
+        - artifact
+      operationId: CopyArtifact
+      parameters:
+        - $ref: '#/parameters/requestId'
+        - $ref: '#/parameters/projectName'
+        - $ref: '#/parameters/repositoryName'
+        - name: from
+          in: query
+          description: The artifact from which the new artifact is copied from, the format should be "project/repository:tag" or "project/repository@digest".
+          type: string
+          required: true
+      responses:
+        '201':
+          $ref: '#/responses/201'
+        '400':
+          $ref: '#/responses/400'
+        '401':
+          $ref: '#/responses/401'
+        '403':
+          $ref: '#/responses/403'
+        '404':
+          $ref: '#/responses/404'
+        '409':
+          $ref: '#/responses/409'
+        '500':
+          $ref: '#/responses/500'
   /projects/{project_name}/repositories/{repository_name}/artifacts/{reference}:
     get:
       summary: Get the specific artifact

src/api/artifact/controller.go

@@ -20,6 +20,7 @@ import (
 	"github.com/goharbor/harbor/src/api/artifact/abstractor"
 	"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
 	"github.com/goharbor/harbor/src/api/artifact/descriptor"
+	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/internal"
 	"github.com/goharbor/harbor/src/pkg/art"
@@ -28,10 +29,10 @@ import (
 	"github.com/goharbor/harbor/src/pkg/immutabletag/match"
 	"github.com/goharbor/harbor/src/pkg/immutabletag/match/rule"
 	"github.com/goharbor/harbor/src/pkg/label"
+	"github.com/goharbor/harbor/src/pkg/registry"
 	"github.com/goharbor/harbor/src/pkg/signature"
 	"github.com/opencontainers/go-digest"
 	"strings"
-
 	// registry image resolvers
 	_ "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver/image"
 	// register chart resolver
@@ -71,6 +72,8 @@ type Controller interface {
 	GetByReference(ctx context.Context, repository, reference string, option *Option) (artifact *Artifact, err error)
 	// Delete the artifact specified by ID. All tags attached to the artifact are deleted as well
 	Delete(ctx context.Context, id int64) (err error)
+	// Copy the artifact whose ID is specified by "srcArtID" into the repository specified by "dstRepoID"
+	Copy(ctx context.Context, srcArtID, dstRepoID int64) (id int64, err error)
 	// ListTags lists the tags according to the query, specify the properties returned with option
 	ListTags(ctx context.Context, query *q.Query, option *TagOption) (tags []*Tag, err error)
 	// CreateTag creates a tag
@@ -101,6 +104,7 @@ func NewController() Controller {
 		labelMgr:     label.Mgr,
 		abstractor:   abstractor.NewAbstractor(),
 		immutableMtr: rule.NewRuleMatcher(),
+		regCli:       registry.Cli,
 	}
 }
 
@@ -115,6 +119,7 @@ type controller struct {
 	labelMgr     label.Manager
 	abstractor   abstractor.Abstractor
 	immutableMtr match.ImmutableTagMatcher
+	regCli       registry.Client
 }
 
 func (c *controller) Ensure(ctx context.Context, repositoryID int64, digest string, tags ...string) (bool, int64, error) {
@@ -374,6 +379,68 @@ func (c *controller) deleteDeeply(ctx context.Context, id int64, isRoot bool) er
 	return nil
 }
 
+func (c *controller) Copy(ctx context.Context, srcArtID, dstRepoID int64) (int64, error) {
+	srcArt, err := c.Get(ctx, srcArtID, &Option{WithTag: true})
+	if err != nil {
+		return 0, err
+	}
+	srcRepo, err := c.repoMgr.Get(ctx, srcArt.RepositoryID)
+	if err != nil {
+		return 0, err
+	}
+	dstRepo, err := c.repoMgr.Get(ctx, dstRepoID)
+	if err != nil {
+		return 0, err
+	}
+
+	_, err = c.artMgr.GetByDigest(ctx, dstRepoID, srcArt.Digest)
+	// the artifact already exists in the destination repository
+	if err == nil {
+		return 0, ierror.New(nil).WithCode(ierror.ConflictCode).
+			WithMessage("the artifact %s already exists under the repository %s",
+				srcArt.Digest, dstRepo.Name)
+	}
+	if !ierror.IsErr(err, ierror.NotFoundCode) {
+		return 0, err
+	}
+
+	// only copy the tags of outermost artifact
+	var tags []string
+	for _, tag := range srcArt.Tags {
+		tags = append(tags, tag.Name)
+	}
+	return c.copyDeeply(ctx, srcRepo, srcArt, dstRepo, tags...)
+}
+
+// as we call the docker registry APIs in the registry client directly,
+// this bypass our own logic(ensure, fire event, etc.) inside the registry handlers,
+// these logic must be covered explicitly here.
+// "copyDeeply" iterates the child artifacts and copy them first
+func (c *controller) copyDeeply(ctx context.Context, srcRepo *models.RepoRecord, srcArt *Artifact,
+	dstRepo *models.RepoRecord, tags ...string) (int64, error) {
+	// copy child artifacts if contains any
+	for _, reference := range srcArt.References {
+		childArt, err := c.Get(ctx, reference.ChildID, nil)
+		if err != nil {
+			return 0, err
+		}
+		if _, err = c.copyDeeply(ctx, srcRepo, childArt, dstRepo); err != nil {
+			return 0, err
+		}
+	}
+	// copy the parent artifact
+	if err := c.regCli.Copy(srcRepo.Name, srcArt.Digest,
+		dstRepo.Name, srcArt.Digest, false); err != nil {
+		return 0, err
+	}
+	_, id, err := c.Ensure(ctx, dstRepo.RepositoryID, srcArt.Digest, tags...)
+	if err != nil {
+		return 0, err
+	}
+	// TODO fire event
+	return id, nil
+}
+
 func (c *controller) CreateTag(ctx context.Context, tag *Tag) (int64, error) {
 	// TODO fire event
 	return c.tagMgr.Create(ctx, &(tag.Tag))

src/api/artifact/controller_test.go

@@ -28,6 +28,7 @@ import (
 	artrashtesting "github.com/goharbor/harbor/src/testing/pkg/artifactrash"
 	immutesting "github.com/goharbor/harbor/src/testing/pkg/immutabletag"
 	"github.com/goharbor/harbor/src/testing/pkg/label"
+	"github.com/goharbor/harbor/src/testing/pkg/registry"
 	repotesting "github.com/goharbor/harbor/src/testing/pkg/repository"
 	tagtesting "github.com/goharbor/harbor/src/testing/pkg/tag"
 	"github.com/stretchr/testify/mock"
@@ -75,6 +76,7 @@ type controllerTestSuite struct {
 	labelMgr     *label.FakeManager
 	abstractor   *fakeAbstractor
 	immutableMtr *immutesting.FakeMatcher
+	regCli       *registry.FakeClient
 }
 
 func (c *controllerTestSuite) SetupTest() {
@@ -85,6 +87,7 @@ func (c *controllerTestSuite) SetupTest() {
 	c.labelMgr = &label.FakeManager{}
 	c.abstractor = &fakeAbstractor{}
 	c.immutableMtr = &immutesting.FakeMatcher{}
+	c.regCli = &registry.FakeClient{}
 	c.ctl = &controller{
 		repoMgr:      c.repoMgr,
 		artMgr:       c.artMgr,
@@ -93,6 +96,7 @@ func (c *controllerTestSuite) SetupTest() {
 		labelMgr:     c.labelMgr,
 		abstractor:   c.abstractor,
 		immutableMtr: c.immutableMtr,
+		regCli:       c.regCli,
 	}
 	descriptor.Register(&fakeDescriptor{}, "")
 }
@@ -491,6 +495,29 @@ func (c *controllerTestSuite) TestDeleteDeeply() {
 	c.Require().Nil(err)
 }
 
+func (c *controllerTestSuite) TestCopy() {
+	c.artMgr.On("Get").Return(&artifact.Artifact{
+		ID: 1,
+	}, nil)
+	c.artMgr.On("GetByDigest").Return(nil, ierror.NotFoundError(nil))
+	c.tagMgr.On("List").Return([]*tag.Tag{
+		{
+			ID:   1,
+			Name: "latest",
+		},
+	}, nil)
+	c.tagMgr.On("Update").Return(nil)
+	c.repoMgr.On("Get").Return(&models.RepoRecord{
+		RepositoryID: 1,
+		Name:         "library/hello-world",
+	}, nil)
+	c.abstractor.On("AbstractMetadata").Return(nil)
+	c.artMgr.On("Create").Return(1, nil)
+	c.regCli.On("Copy").Return(nil)
+	_, err := c.ctl.Copy(nil, 1, 1)
+	c.Require().Nil(err)
+}
+
 func (c *controllerTestSuite) TestListTags() {
 	c.tagMgr.On("List").Return([]*tag.Tag{
 		{

src/common/rbac/project/visitor_role.go

@@ -128,6 +128,7 @@ var (
 			{Resource: rbac.ResourceScanner, Action: rbac.ActionRead},
 			{Resource: rbac.ResourceScanner, Action: rbac.ActionCreate},
 
+			{Resource: rbac.ResourceArtifact, Action: rbac.ActionCreate},
 			{Resource: rbac.ResourceArtifact, Action: rbac.ActionRead},
 			{Resource: rbac.ResourceArtifact, Action: rbac.ActionDelete},
 			{Resource: rbac.ResourceArtifact, Action: rbac.ActionList},
@@ -228,6 +229,7 @@ var (
 
 			{Resource: rbac.ResourceScanner, Action: rbac.ActionRead},
 
+			{Resource: rbac.ResourceArtifact, Action: rbac.ActionCreate},
 			{Resource: rbac.ResourceArtifact, Action: rbac.ActionRead},
 			{Resource: rbac.ResourceArtifact, Action: rbac.ActionDelete},
 			{Resource: rbac.ResourceArtifact, Action: rbac.ActionList},
@@ -295,6 +297,7 @@ var (
 
 			{Resource: rbac.ResourceScanner, Action: rbac.ActionRead},
 
+			{Resource: rbac.ResourceArtifact, Action: rbac.ActionCreate},
 			{Resource: rbac.ResourceArtifact, Action: rbac.ActionRead},
 			{Resource: rbac.ResourceArtifact, Action: rbac.ActionList},
 			{Resource: rbac.ResourceArtifactAddition, Action: rbac.ActionRead},

src/pkg/registry/client.go

@@ -0,0 +1,165 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package registry
+
+import (
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/goharbor/harbor/src/common/utils/registry"
+	"github.com/goharbor/harbor/src/common/utils/registry/auth"
+	"github.com/goharbor/harbor/src/core/config"
+	ierror "github.com/goharbor/harbor/src/internal/error"
+	"github.com/goharbor/harbor/src/replication/util"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"net/http"
+)
+
+// TODO we'll merge all registry related code into this package before releasing 2.0
+
+var (
+	// Cli is the global registry client instance, it targets to the backend docker registry
+	Cli = func() Client {
+		url, _ := config.RegistryURL()
+		username, password := config.RegistryCredential()
+		return NewClient(url, true, username, password)
+	}()
+
+	accepts = []string{
+		v1.MediaTypeImageIndex,
+		manifestlist.MediaTypeManifestList,
+		v1.MediaTypeImageManifest,
+		schema2.MediaTypeManifest,
+		schema1.MediaTypeSignedManifest,
+	}
+)
+
+// Client defines the methods that a registry client should implements
+type Client interface {
+	// Copy the artifact from source repository to the destination. The "override"
+	// is used to specify whether the destination artifact will be overridden if
+	// its name is same with source but digest isn't
+	Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) (err error)
+	// TODO defines other methods
+}
+
+// NewClient creates a registry client based on the provided information
+// TODO support HTTPS
+func NewClient(url string, insecure bool, username, password string) Client {
+	transport := util.GetHTTPTransport(insecure)
+	authorizer := auth.NewAuthorizer(auth.NewBasicAuthCredential(username, password),
+		&http.Client{
+			Transport: transport,
+		})
+	return &client{
+		url: url,
+		client: &http.Client{
+			Transport: registry.NewTransport(transport, authorizer),
+		},
+	}
+}
+
+type client struct {
+	url    string
+	client *http.Client
+}
+
+// TODO extend this method to support copy artifacts between different registries when merging codes
+// TODO this can be used in replication to replace the existing implementation
+// TODO add unit test case
+func (c *client) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) error {
+	src, err := registry.NewRepository(srcRepo, c.url, c.client)
+	if err != nil {
+		return err
+	}
+	dst, err := registry.NewRepository(dstRepo, c.url, c.client)
+	if err != nil {
+		return err
+	}
+	// pull the manifest from the source repository
+	srcDgt, mediaType, payload, err := src.PullManifest(srcRef, accepts)
+	if err != nil {
+		return err
+	}
+
+	// check the existence of the artifact on the destination repository
+	dstDgt, exist, err := dst.ManifestExist(dstRef)
+	if err != nil {
+		return err
+	}
+	if exist {
+		// the same artifact already exists
+		if srcDgt == dstDgt {
+			return nil
+		}
+		// the same name artifact exists, but not allowed to override
+		if !override {
+			return ierror.New(nil).WithCode(ierror.PreconditionCode).
+				WithMessage("the same name but different digest artifact exists, but the override is set to false")
+		}
+	}
+
+	manifest, _, err := registry.UnMarshal(mediaType, payload)
+	if err != nil {
+		return err
+	}
+	for _, descriptor := range manifest.References() {
+		digest := descriptor.Digest.String()
+		switch descriptor.MediaType {
+		// skip foreign layer
+		case schema2.MediaTypeForeignLayer:
+			continue
+		// manifest or index
+		case v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList,
+			v1.MediaTypeImageManifest, schema2.MediaTypeManifest,
+			schema1.MediaTypeSignedManifest:
+			if err = c.Copy(srcRepo, digest, dstRepo, digest, false); err != nil {
+				return err
+			}
+		// common layer
+		default:
+			exist, err := dst.BlobExist(digest)
+			if err != nil {
+				return err
+			}
+			// the layer already exist, skip
+			if exist {
+				continue
+			}
+			// when the copy happens inside the same registry, use mount
+			if err = dst.MountBlob(digest, srcRepo); err != nil {
+				return err
+			}
+			/*
+				// copy happens between different registries
+				size, data, err := src.PullBlob(digest)
+				if err != nil {
+					return err
+				}
+				defer data.Close()
+				if err = dst.PushBlob(digest, size, data); err != nil {
+					return err
+				}
+			*/
+		}
+	}
+
+	// push manifest to the destination repository
+	if _, err = dst.PushManifest(dstRef, mediaType, payload); err != nil {
+		return err
+	}
+
+	return nil
+}

src/server/v2.0/handler/artifact.go

@@ -17,16 +17,19 @@ package handler
 import (
 	"context"
 	"fmt"
+	"github.com/docker/distribution/reference"
 	"github.com/go-openapi/runtime"
 	"github.com/go-openapi/runtime/middleware"
 	"github.com/goharbor/harbor/src/api/artifact"
+	"github.com/goharbor/harbor/src/api/repository"
 	"github.com/goharbor/harbor/src/common/rbac"
+	"github.com/goharbor/harbor/src/common/utils"
 	ierror "github.com/goharbor/harbor/src/internal/error"
 	"github.com/goharbor/harbor/src/pkg/project"
 	"github.com/goharbor/harbor/src/pkg/q"
-	"github.com/goharbor/harbor/src/pkg/repository"
 	"github.com/goharbor/harbor/src/server/v2.0/handler/model"
 	operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/artifact"
+	"github.com/opencontainers/go-digest"
 	"net/http"
 	"strings"
 	"time"
@@ -36,7 +39,7 @@ func newArtifactAPI() *artifactAPI {
 	return &artifactAPI{
 		artCtl:  artifact.Ctl,
 		proMgr:  project.Mgr,
-		repoMgr: repository.Mgr,
+		repoCtl: repository.Ctl,
 	}
 }
 
@@ -44,7 +47,7 @@ type artifactAPI struct {
 	BaseAPI
 	artCtl  artifact.Controller
 	proMgr  project.Manager
-	repoMgr repository.Manager
+	repoCtl repository.Controller
 }
 
 func (a *artifactAPI) ListArtifacts(ctx context.Context, params operation.ListArtifactsParams) middleware.Responder {
@@ -67,7 +70,7 @@ func (a *artifactAPI) ListArtifacts(ctx context.Context, params operation.ListAr
 	if params.PageSize != nil {
 		query.PageSize = *(params.PageSize)
 	}
-	repository, err := a.repoMgr.GetByName(ctx, fmt.Sprintf("%s/%s", params.ProjectName, params.RepositoryName))
+	repository, err := a.repoCtl.GetByName(ctx, fmt.Sprintf("%s/%s", params.ProjectName, params.RepositoryName))
 	if err != nil {
 		return a.SendError(ctx, err)
 	}
@@ -133,6 +136,56 @@ func (a *artifactAPI) DeleteArtifact(ctx context.Context, params operation.Delet
 	return operation.NewDeleteArtifactOK()
 }
 
+// TODO immutable, quota, readonly middlewares should cover this API
+func (a *artifactAPI) CopyArtifact(ctx context.Context, params operation.CopyArtifactParams) middleware.Responder {
+	if err := a.RequireProjectAccess(ctx, params.ProjectName, rbac.ActionCreate, rbac.ResourceArtifact); err != nil {
+		return a.SendError(ctx, err)
+	}
+	srcRepo, srcRef, err := parse(params.From)
+	if err != nil {
+		return a.SendError(ctx, err)
+	}
+	srcPro, _ := utils.ParseRepository(srcRepo)
+	if err = a.RequireProjectAccess(ctx, srcPro, rbac.ActionRead, rbac.ResourceArtifact); err != nil {
+		return a.SendError(ctx, err)
+	}
+	srcArt, err := a.artCtl.GetByReference(ctx, srcRepo, srcRef, &artifact.Option{WithTag: true})
+	if err != nil {
+		return a.SendError(ctx, err)
+	}
+	_, id, err := a.repoCtl.Ensure(ctx, params.ProjectName+"/"+params.RepositoryName)
+	if err != nil {
+		return a.SendError(ctx, err)
+	}
+	id, err = a.artCtl.Copy(ctx, srcArt.ID, id)
+	if err != nil {
+		return a.SendError(ctx, err)
+	}
+	// TODO set location header
+	_ = id
+	return operation.NewCopyArtifactCreated()
+}
+
+// parse "repository:tag" or "repository@digest" into repository and reference parts
+func parse(s string) (string, string, error) {
+	matches := reference.ReferenceRegexp.FindStringSubmatch(s)
+	if matches == nil {
+		return "", "", ierror.New(nil).WithCode(ierror.BadRequestCode).
+			WithMessage("invalid input: %s", s)
+	}
+	repository := matches[1]
+	reference := matches[2]
+	if matches[3] != "" {
+		_, err := digest.Parse(matches[3])
+		if err != nil {
+			return "", "", ierror.New(nil).WithCode(ierror.BadRequestCode).
+				WithMessage("invalid input: %s", s)
+		}
+		reference = matches[3]
+	}
+	return repository, reference, nil
+}
+
 func (a *artifactAPI) CreateTag(ctx context.Context, params operation.CreateTagParams) middleware.Responder {
 	if err := a.RequireProjectAccess(ctx, params.ProjectName, rbac.ActionCreate, rbac.ResourceTag); err != nil {
 		return a.SendError(ctx, err)

src/server/v2.0/handler/artifact_test.go

@@ -0,0 +1,52 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handler
+
+import (
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestParse(t *testing.T) {
+	// with tag
+	input := "library/hello-world:latest"
+	repository, reference, err := parse(input)
+	require.Nil(t, err)
+	assert.Equal(t, "library/hello-world", repository)
+	assert.Equal(t, "latest", reference)
+
+	// with digest
+	input = "library/hello-world@sha256:9572f7cdcee8591948c2963463447a53466950b3fc15a247fcad1917ca215a2f"
+	repository, reference, err = parse(input)
+	require.Nil(t, err)
+	assert.Equal(t, "library/hello-world", repository)
+	assert.Equal(t, "sha256:9572f7cdcee8591948c2963463447a53466950b3fc15a247fcad1917ca215a2f", reference)
+
+	// invalid digest
+	input = "library/hello-world@sha256:invalid_digest"
+	repository, reference, err = parse(input)
+	require.NotNil(t, err)
+
+	// invalid character
+	input = "library/hello-world?#:latest"
+	repository, reference, err = parse(input)
+	require.NotNil(t, err)
+
+	// empty input
+	input = ""
+	repository, reference, err = parse(input)
+	require.NotNil(t, err)
+}

src/testing/api/artifact/controller.go

@@ -76,6 +76,12 @@ func (f *FakeController) Delete(ctx context.Context, id int64) (err error) {
 	return args.Error(0)
 }
 
+// Copy ...
+func (f *FakeController) Copy(ctx context.Context, srcArtID, dstRepoID int64) (int64, error) {
+	args := f.Called()
+	return int64(args.Int(0)), args.Error(1)
+}
+
 // ListTags ...
 func (f *FakeController) ListTags(ctx context.Context, query *q.Query, option *artifact.TagOption) ([]*artifact.Tag, error) {
 	args := f.Called()

src/testing/pkg/registry/client.go

@@ -0,0 +1,30 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package registry
+
+import (
+	"github.com/stretchr/testify/mock"
+)
+
+// FakeClient is a fake registry client that implement src/pkg/registry.Client interface
+type FakeClient struct {
+	mock.Mock
+}
+
+// Copy ...
+func (f *FakeClient) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) error {
+	args := f.Called()
+	return args.Error(0)
+}