Implement copy artifact API

Copy artifact into the repository from the specified artifact

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2020-02-13 16:04:47 +08:00
parent beddef6873
commit c4d4850845
9 changed files with 438 additions and 5 deletions

View File

@ -187,6 +187,36 @@ paths:
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
post:
summary: Copy artifact
description: Copy the artifact specified in the "from" parameter to the repository.
tags:
- artifact
operationId: CopyArtifact
parameters:
- $ref: '#/parameters/requestId'
- $ref: '#/parameters/projectName'
- $ref: '#/parameters/repositoryName'
- name: from
in: query
description: The artifact from which the new artifact is copied from, the format should be "project/repository:tag" or "project/repository@digest".
type: string
required: true
responses:
'201':
$ref: '#/responses/201'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'409':
$ref: '#/responses/409'
'500':
$ref: '#/responses/500'
/projects/{project_name}/repositories/{repository_name}/artifacts/{reference}:
get:
summary: Get the specific artifact

View File

@ -20,6 +20,7 @@ import (
"github.com/goharbor/harbor/src/api/artifact/abstractor"
"github.com/goharbor/harbor/src/api/artifact/abstractor/resolver"
"github.com/goharbor/harbor/src/api/artifact/descriptor"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/internal"
"github.com/goharbor/harbor/src/pkg/art"
@ -28,10 +29,10 @@ import (
"github.com/goharbor/harbor/src/pkg/immutabletag/match"
"github.com/goharbor/harbor/src/pkg/immutabletag/match/rule"
"github.com/goharbor/harbor/src/pkg/label"
"github.com/goharbor/harbor/src/pkg/registry"
"github.com/goharbor/harbor/src/pkg/signature"
"github.com/opencontainers/go-digest"
"strings"
// registry image resolvers
_ "github.com/goharbor/harbor/src/api/artifact/abstractor/resolver/image"
// register chart resolver
@ -71,6 +72,8 @@ type Controller interface {
GetByReference(ctx context.Context, repository, reference string, option *Option) (artifact *Artifact, err error)
// Delete the artifact specified by ID. All tags attached to the artifact are deleted as well
Delete(ctx context.Context, id int64) (err error)
// Copy the artifact whose ID is specified by "srcArtID" into the repository specified by "dstRepoID"
Copy(ctx context.Context, srcArtID, dstRepoID int64) (id int64, err error)
// ListTags lists the tags according to the query, specify the properties returned with option
ListTags(ctx context.Context, query *q.Query, option *TagOption) (tags []*Tag, err error)
// CreateTag creates a tag
@ -101,6 +104,7 @@ func NewController() Controller {
labelMgr: label.Mgr,
abstractor: abstractor.NewAbstractor(),
immutableMtr: rule.NewRuleMatcher(),
regCli: registry.Cli,
}
}
@ -115,6 +119,7 @@ type controller struct {
labelMgr label.Manager
abstractor abstractor.Abstractor
immutableMtr match.ImmutableTagMatcher
regCli registry.Client
}
func (c *controller) Ensure(ctx context.Context, repositoryID int64, digest string, tags ...string) (bool, int64, error) {
@ -374,6 +379,68 @@ func (c *controller) deleteDeeply(ctx context.Context, id int64, isRoot bool) er
return nil
}
func (c *controller) Copy(ctx context.Context, srcArtID, dstRepoID int64) (int64, error) {
srcArt, err := c.Get(ctx, srcArtID, &Option{WithTag: true})
if err != nil {
return 0, err
}
srcRepo, err := c.repoMgr.Get(ctx, srcArt.RepositoryID)
if err != nil {
return 0, err
}
dstRepo, err := c.repoMgr.Get(ctx, dstRepoID)
if err != nil {
return 0, err
}
_, err = c.artMgr.GetByDigest(ctx, dstRepoID, srcArt.Digest)
// the artifact already exists in the destination repository
if err == nil {
return 0, ierror.New(nil).WithCode(ierror.ConflictCode).
WithMessage("the artifact %s already exists under the repository %s",
srcArt.Digest, dstRepo.Name)
}
if !ierror.IsErr(err, ierror.NotFoundCode) {
return 0, err
}
// only copy the tags of outermost artifact
var tags []string
for _, tag := range srcArt.Tags {
tags = append(tags, tag.Name)
}
return c.copyDeeply(ctx, srcRepo, srcArt, dstRepo, tags...)
}
// as we call the docker registry APIs in the registry client directly,
// this bypass our own logic(ensure, fire event, etc.) inside the registry handlers,
// these logic must be covered explicitly here.
// "copyDeeply" iterates the child artifacts and copy them first
func (c *controller) copyDeeply(ctx context.Context, srcRepo *models.RepoRecord, srcArt *Artifact,
dstRepo *models.RepoRecord, tags ...string) (int64, error) {
// copy child artifacts if contains any
for _, reference := range srcArt.References {
childArt, err := c.Get(ctx, reference.ChildID, nil)
if err != nil {
return 0, err
}
if _, err = c.copyDeeply(ctx, srcRepo, childArt, dstRepo); err != nil {
return 0, err
}
}
// copy the parent artifact
if err := c.regCli.Copy(srcRepo.Name, srcArt.Digest,
dstRepo.Name, srcArt.Digest, false); err != nil {
return 0, err
}
_, id, err := c.Ensure(ctx, dstRepo.RepositoryID, srcArt.Digest, tags...)
if err != nil {
return 0, err
}
// TODO fire event
return id, nil
}
func (c *controller) CreateTag(ctx context.Context, tag *Tag) (int64, error) {
// TODO fire event
return c.tagMgr.Create(ctx, &(tag.Tag))

View File

@ -28,6 +28,7 @@ import (
artrashtesting "github.com/goharbor/harbor/src/testing/pkg/artifactrash"
immutesting "github.com/goharbor/harbor/src/testing/pkg/immutabletag"
"github.com/goharbor/harbor/src/testing/pkg/label"
"github.com/goharbor/harbor/src/testing/pkg/registry"
repotesting "github.com/goharbor/harbor/src/testing/pkg/repository"
tagtesting "github.com/goharbor/harbor/src/testing/pkg/tag"
"github.com/stretchr/testify/mock"
@ -75,6 +76,7 @@ type controllerTestSuite struct {
labelMgr *label.FakeManager
abstractor *fakeAbstractor
immutableMtr *immutesting.FakeMatcher
regCli *registry.FakeClient
}
func (c *controllerTestSuite) SetupTest() {
@ -85,6 +87,7 @@ func (c *controllerTestSuite) SetupTest() {
c.labelMgr = &label.FakeManager{}
c.abstractor = &fakeAbstractor{}
c.immutableMtr = &immutesting.FakeMatcher{}
c.regCli = &registry.FakeClient{}
c.ctl = &controller{
repoMgr: c.repoMgr,
artMgr: c.artMgr,
@ -93,6 +96,7 @@ func (c *controllerTestSuite) SetupTest() {
labelMgr: c.labelMgr,
abstractor: c.abstractor,
immutableMtr: c.immutableMtr,
regCli: c.regCli,
}
descriptor.Register(&fakeDescriptor{}, "")
}
@ -491,6 +495,29 @@ func (c *controllerTestSuite) TestDeleteDeeply() {
c.Require().Nil(err)
}
func (c *controllerTestSuite) TestCopy() {
c.artMgr.On("Get").Return(&artifact.Artifact{
ID: 1,
}, nil)
c.artMgr.On("GetByDigest").Return(nil, ierror.NotFoundError(nil))
c.tagMgr.On("List").Return([]*tag.Tag{
{
ID: 1,
Name: "latest",
},
}, nil)
c.tagMgr.On("Update").Return(nil)
c.repoMgr.On("Get").Return(&models.RepoRecord{
RepositoryID: 1,
Name: "library/hello-world",
}, nil)
c.abstractor.On("AbstractMetadata").Return(nil)
c.artMgr.On("Create").Return(1, nil)
c.regCli.On("Copy").Return(nil)
_, err := c.ctl.Copy(nil, 1, 1)
c.Require().Nil(err)
}
func (c *controllerTestSuite) TestListTags() {
c.tagMgr.On("List").Return([]*tag.Tag{
{

View File

@ -128,6 +128,7 @@ var (
{Resource: rbac.ResourceScanner, Action: rbac.ActionRead},
{Resource: rbac.ResourceScanner, Action: rbac.ActionCreate},
{Resource: rbac.ResourceArtifact, Action: rbac.ActionCreate},
{Resource: rbac.ResourceArtifact, Action: rbac.ActionRead},
{Resource: rbac.ResourceArtifact, Action: rbac.ActionDelete},
{Resource: rbac.ResourceArtifact, Action: rbac.ActionList},
@ -228,6 +229,7 @@ var (
{Resource: rbac.ResourceScanner, Action: rbac.ActionRead},
{Resource: rbac.ResourceArtifact, Action: rbac.ActionCreate},
{Resource: rbac.ResourceArtifact, Action: rbac.ActionRead},
{Resource: rbac.ResourceArtifact, Action: rbac.ActionDelete},
{Resource: rbac.ResourceArtifact, Action: rbac.ActionList},
@ -295,6 +297,7 @@ var (
{Resource: rbac.ResourceScanner, Action: rbac.ActionRead},
{Resource: rbac.ResourceArtifact, Action: rbac.ActionCreate},
{Resource: rbac.ResourceArtifact, Action: rbac.ActionRead},
{Resource: rbac.ResourceArtifact, Action: rbac.ActionList},
{Resource: rbac.ResourceArtifactAddition, Action: rbac.ActionRead},

165
src/pkg/registry/client.go Normal file
View File

@ -0,0 +1,165 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/core/config"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/replication/util"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"net/http"
)
// TODO we'll merge all registry related code into this package before releasing 2.0
var (
// Cli is the global registry client instance, it targets to the backend docker registry
Cli = func() Client {
url, _ := config.RegistryURL()
username, password := config.RegistryCredential()
return NewClient(url, true, username, password)
}()
accepts = []string{
v1.MediaTypeImageIndex,
manifestlist.MediaTypeManifestList,
v1.MediaTypeImageManifest,
schema2.MediaTypeManifest,
schema1.MediaTypeSignedManifest,
}
)
// Client defines the methods that a registry client should implements
type Client interface {
// Copy the artifact from source repository to the destination. The "override"
// is used to specify whether the destination artifact will be overridden if
// its name is same with source but digest isn't
Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) (err error)
// TODO defines other methods
}
// NewClient creates a registry client based on the provided information
// TODO support HTTPS
func NewClient(url string, insecure bool, username, password string) Client {
transport := util.GetHTTPTransport(insecure)
authorizer := auth.NewAuthorizer(auth.NewBasicAuthCredential(username, password),
&http.Client{
Transport: transport,
})
return &client{
url: url,
client: &http.Client{
Transport: registry.NewTransport(transport, authorizer),
},
}
}
type client struct {
url string
client *http.Client
}
// TODO extend this method to support copy artifacts between different registries when merging codes
// TODO this can be used in replication to replace the existing implementation
// TODO add unit test case
func (c *client) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) error {
src, err := registry.NewRepository(srcRepo, c.url, c.client)
if err != nil {
return err
}
dst, err := registry.NewRepository(dstRepo, c.url, c.client)
if err != nil {
return err
}
// pull the manifest from the source repository
srcDgt, mediaType, payload, err := src.PullManifest(srcRef, accepts)
if err != nil {
return err
}
// check the existence of the artifact on the destination repository
dstDgt, exist, err := dst.ManifestExist(dstRef)
if err != nil {
return err
}
if exist {
// the same artifact already exists
if srcDgt == dstDgt {
return nil
}
// the same name artifact exists, but not allowed to override
if !override {
return ierror.New(nil).WithCode(ierror.PreconditionCode).
WithMessage("the same name but different digest artifact exists, but the override is set to false")
}
}
manifest, _, err := registry.UnMarshal(mediaType, payload)
if err != nil {
return err
}
for _, descriptor := range manifest.References() {
digest := descriptor.Digest.String()
switch descriptor.MediaType {
// skip foreign layer
case schema2.MediaTypeForeignLayer:
continue
// manifest or index
case v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList,
v1.MediaTypeImageManifest, schema2.MediaTypeManifest,
schema1.MediaTypeSignedManifest:
if err = c.Copy(srcRepo, digest, dstRepo, digest, false); err != nil {
return err
}
// common layer
default:
exist, err := dst.BlobExist(digest)
if err != nil {
return err
}
// the layer already exist, skip
if exist {
continue
}
// when the copy happens inside the same registry, use mount
if err = dst.MountBlob(digest, srcRepo); err != nil {
return err
}
/*
// copy happens between different registries
size, data, err := src.PullBlob(digest)
if err != nil {
return err
}
defer data.Close()
if err = dst.PushBlob(digest, size, data); err != nil {
return err
}
*/
}
}
// push manifest to the destination repository
if _, err = dst.PushManifest(dstRef, mediaType, payload); err != nil {
return err
}
return nil
}

View File

@ -17,16 +17,19 @@ package handler
import (
"context"
"fmt"
"github.com/docker/distribution/reference"
"github.com/go-openapi/runtime"
"github.com/go-openapi/runtime/middleware"
"github.com/goharbor/harbor/src/api/artifact"
"github.com/goharbor/harbor/src/api/repository"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/utils"
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/project"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/repository"
"github.com/goharbor/harbor/src/server/v2.0/handler/model"
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/artifact"
"github.com/opencontainers/go-digest"
"net/http"
"strings"
"time"
@ -36,7 +39,7 @@ func newArtifactAPI() *artifactAPI {
return &artifactAPI{
artCtl: artifact.Ctl,
proMgr: project.Mgr,
repoMgr: repository.Mgr,
repoCtl: repository.Ctl,
}
}
@ -44,7 +47,7 @@ type artifactAPI struct {
BaseAPI
artCtl artifact.Controller
proMgr project.Manager
repoMgr repository.Manager
repoCtl repository.Controller
}
func (a *artifactAPI) ListArtifacts(ctx context.Context, params operation.ListArtifactsParams) middleware.Responder {
@ -67,7 +70,7 @@ func (a *artifactAPI) ListArtifacts(ctx context.Context, params operation.ListAr
if params.PageSize != nil {
query.PageSize = *(params.PageSize)
}
repository, err := a.repoMgr.GetByName(ctx, fmt.Sprintf("%s/%s", params.ProjectName, params.RepositoryName))
repository, err := a.repoCtl.GetByName(ctx, fmt.Sprintf("%s/%s", params.ProjectName, params.RepositoryName))
if err != nil {
return a.SendError(ctx, err)
}
@ -133,6 +136,56 @@ func (a *artifactAPI) DeleteArtifact(ctx context.Context, params operation.Delet
return operation.NewDeleteArtifactOK()
}
// TODO immutable, quota, readonly middlewares should cover this API
func (a *artifactAPI) CopyArtifact(ctx context.Context, params operation.CopyArtifactParams) middleware.Responder {
if err := a.RequireProjectAccess(ctx, params.ProjectName, rbac.ActionCreate, rbac.ResourceArtifact); err != nil {
return a.SendError(ctx, err)
}
srcRepo, srcRef, err := parse(params.From)
if err != nil {
return a.SendError(ctx, err)
}
srcPro, _ := utils.ParseRepository(srcRepo)
if err = a.RequireProjectAccess(ctx, srcPro, rbac.ActionRead, rbac.ResourceArtifact); err != nil {
return a.SendError(ctx, err)
}
srcArt, err := a.artCtl.GetByReference(ctx, srcRepo, srcRef, &artifact.Option{WithTag: true})
if err != nil {
return a.SendError(ctx, err)
}
_, id, err := a.repoCtl.Ensure(ctx, params.ProjectName+"/"+params.RepositoryName)
if err != nil {
return a.SendError(ctx, err)
}
id, err = a.artCtl.Copy(ctx, srcArt.ID, id)
if err != nil {
return a.SendError(ctx, err)
}
// TODO set location header
_ = id
return operation.NewCopyArtifactCreated()
}
// parse "repository:tag" or "repository@digest" into repository and reference parts
func parse(s string) (string, string, error) {
matches := reference.ReferenceRegexp.FindStringSubmatch(s)
if matches == nil {
return "", "", ierror.New(nil).WithCode(ierror.BadRequestCode).
WithMessage("invalid input: %s", s)
}
repository := matches[1]
reference := matches[2]
if matches[3] != "" {
_, err := digest.Parse(matches[3])
if err != nil {
return "", "", ierror.New(nil).WithCode(ierror.BadRequestCode).
WithMessage("invalid input: %s", s)
}
reference = matches[3]
}
return repository, reference, nil
}
func (a *artifactAPI) CreateTag(ctx context.Context, params operation.CreateTagParams) middleware.Responder {
if err := a.RequireProjectAccess(ctx, params.ProjectName, rbac.ActionCreate, rbac.ResourceTag); err != nil {
return a.SendError(ctx, err)

View File

@ -0,0 +1,52 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package handler
import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing"
)
func TestParse(t *testing.T) {
// with tag
input := "library/hello-world:latest"
repository, reference, err := parse(input)
require.Nil(t, err)
assert.Equal(t, "library/hello-world", repository)
assert.Equal(t, "latest", reference)
// with digest
input = "library/hello-world@sha256:9572f7cdcee8591948c2963463447a53466950b3fc15a247fcad1917ca215a2f"
repository, reference, err = parse(input)
require.Nil(t, err)
assert.Equal(t, "library/hello-world", repository)
assert.Equal(t, "sha256:9572f7cdcee8591948c2963463447a53466950b3fc15a247fcad1917ca215a2f", reference)
// invalid digest
input = "library/hello-world@sha256:invalid_digest"
repository, reference, err = parse(input)
require.NotNil(t, err)
// invalid character
input = "library/hello-world?#:latest"
repository, reference, err = parse(input)
require.NotNil(t, err)
// empty input
input = ""
repository, reference, err = parse(input)
require.NotNil(t, err)
}

View File

@ -76,6 +76,12 @@ func (f *FakeController) Delete(ctx context.Context, id int64) (err error) {
return args.Error(0)
}
// Copy ...
func (f *FakeController) Copy(ctx context.Context, srcArtID, dstRepoID int64) (int64, error) {
args := f.Called()
return int64(args.Int(0)), args.Error(1)
}
// ListTags ...
func (f *FakeController) ListTags(ctx context.Context, query *q.Query, option *artifact.TagOption) ([]*artifact.Tag, error) {
args := f.Called()

View File

@ -0,0 +1,30 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"github.com/stretchr/testify/mock"
)
// FakeClient is a fake registry client that implement src/pkg/registry.Client interface
type FakeClient struct {
mock.Mock
}
// Copy ...
func (f *FakeClient) Copy(srcRepo, srcRef, dstRepo, dstRef string, override bool) error {
args := f.Called()
return args.Error(0)
}