From 5219073c498d7182c5587d032ee3aa0e430be4f7 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 2 Apr 2019 16:18:57 +0800 Subject: [PATCH] Call Harbor API to delete the images in Harbor adapter Call Harbor API to delete the images in Harbor adapter to avoid the inconsistent between the different versions of Harbor Signed-off-by: Wenkai Yin --- src/core/api/repository.go | 28 ++++++----- .../ng/adapter/harbor/image_registry.go | 7 +++ src/replication/ng/adapter/image_registry.go | 28 ++++++++--- .../ng/adapter/image_registry_test.go | 46 +++++++++++++++++++ .../ng/transfer/repository/transfer.go | 15 ++---- 5 files changed, 95 insertions(+), 29 deletions(-) create mode 100644 src/replication/ng/adapter/image_registry_test.go diff --git a/src/core/api/repository.go b/src/core/api/repository.go index f790fc45f..59f87f200 100644 --- a/src/core/api/repository.go +++ b/src/core/api/repository.go @@ -37,10 +37,10 @@ import ( "github.com/goharbor/harbor/src/common/utils/notary" "github.com/goharbor/harbor/src/common/utils/registry" "github.com/goharbor/harbor/src/core/config" - "github.com/goharbor/harbor/src/core/notifier" coreutils "github.com/goharbor/harbor/src/core/utils" - "github.com/goharbor/harbor/src/replication/event/notification" - "github.com/goharbor/harbor/src/replication/event/topic" + "github.com/goharbor/harbor/src/replication/ng" + "github.com/goharbor/harbor/src/replication/ng/event" + "github.com/goharbor/harbor/src/replication/ng/model" ) // RepositoryAPI handles request to /api/repositories /api/repositories/tags /api/repositories/manifests, the parm has to be put @@ -328,15 +328,21 @@ func (ra *RepositoryAPI) Delete() { log.Infof("delete tag: %s:%s", repoName, t) go func(tag string) { - image := repoName + ":" + tag - err := notifier.Publish(topic.ReplicationEventTopicOnDeletion, notification.OnDeletionNotification{ - Image: image, - }) - if err != nil { - log.Errorf("failed to publish on deletion topic for resource %s: %v", image, err) - return + e := &event.Event{ + Type: event.EventTypeImagePush, + Resource: &model.Resource{ + Type: model.ResourceTypeRepository, + Metadata: &model.ResourceMetadata{ + Name: repoName, + Namespace: projectName, + Vtags: []string{tag}, + }, + Deleted: true, + }, + } + if err := ng.EventHandler.Handle(e); err != nil { + log.Errorf("failed to handle event: %v", err) } - log.Debugf("the on deletion topic for resource %s published", image) }(t) go func(tag string) { diff --git a/src/replication/ng/adapter/harbor/image_registry.go b/src/replication/ng/adapter/harbor/image_registry.go index d73135929..def13b2e0 100644 --- a/src/replication/ng/adapter/harbor/image_registry.go +++ b/src/replication/ng/adapter/harbor/image_registry.go @@ -66,3 +66,10 @@ func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]* return resources, nil } + +// override the default implementation from the default image registry +// by calling Harbor API directly +func (a *adapter) DeleteManifest(repository, reference string) error { + url := fmt.Sprintf("%s/api/repositories/%s/tags/%s", a.coreServiceURL, repository, reference) + return a.client.Delete(url) +} diff --git a/src/replication/ng/adapter/image_registry.go b/src/replication/ng/adapter/image_registry.go index 264a72228..c786e7210 100644 --- a/src/replication/ng/adapter/image_registry.go +++ b/src/replication/ng/adapter/image_registry.go @@ -22,15 +22,15 @@ import ( "strings" "sync" - "github.com/goharbor/harbor/src/replication/ng/util" - "github.com/docker/distribution" "github.com/docker/distribution/manifest/schema1" "github.com/goharbor/harbor/src/common/http/modifier" common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth" + "github.com/goharbor/harbor/src/common/utils/log" registry_pkg "github.com/goharbor/harbor/src/common/utils/registry" "github.com/goharbor/harbor/src/common/utils/registry/auth" "github.com/goharbor/harbor/src/replication/ng/model" + "github.com/goharbor/harbor/src/replication/ng/util" ) // const definition @@ -45,7 +45,8 @@ type ImageRegistry interface { ManifestExist(repository, reference string) (exist bool, digest string, err error) PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error) PushManifest(repository, reference, mediaType string, payload []byte) error - DeleteManifest(repository, digest string) error + // the "reference" can be "tag" or "digest", the function needs to handle both + DeleteManifest(repository, reference string) error BlobExist(repository, digest string) (exist bool, err error) PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error) PushBlob(repository, digest string, size int64, blob io.Reader) error @@ -179,15 +180,24 @@ func (d *DefaultImageRegistry) PushManifest(repository, reference, mediaType str return err } -// TODO monitor the registry API request in core directly rather than using -// the web hook - // DeleteManifest ... -func (d *DefaultImageRegistry) DeleteManifest(repository, digest string) error { +func (d *DefaultImageRegistry) DeleteManifest(repository, reference string) error { client, err := d.getClient(repository) if err != nil { return err } + digest := reference + if !isDigest(digest) { + dgt, exist, err := client.ManifestExist(reference) + if err != nil { + return err + } + if !exist { + log.Debugf("the manifest of %s:%s doesn't exist", repository, reference) + return nil + } + digest = dgt + } return client.DeleteManifest(digest) } @@ -217,3 +227,7 @@ func (d *DefaultImageRegistry) PushBlob(repository, digest string, size int64, b } return client.PushBlob(digest, size, blob) } + +func isDigest(str string) bool { + return strings.Contains(str, ":") +} diff --git a/src/replication/ng/adapter/image_registry_test.go b/src/replication/ng/adapter/image_registry_test.go new file mode 100644 index 000000000..157471d41 --- /dev/null +++ b/src/replication/ng/adapter/image_registry_test.go @@ -0,0 +1,46 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package adapter + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TODO add UT + +func TestIsDigest(t *testing.T) { + cases := []struct { + str string + isDigest bool + }{ + { + str: "", + isDigest: false, + }, + { + str: "latest", + isDigest: false, + }, + { + str: "sha256:fea8895f450959fa676bcc1df0611ea93823a735a01205fd8622846041d0c7cf", + isDigest: true, + }, + } + for _, c := range cases { + assert.Equal(t, c.isDigest, isDigest(c.str)) + } +} diff --git a/src/replication/ng/transfer/repository/transfer.go b/src/replication/ng/transfer/repository/transfer.go index 891098fdd..8719e0d8b 100644 --- a/src/replication/ng/transfer/repository/transfer.go +++ b/src/replication/ng/transfer/repository/transfer.go @@ -278,10 +278,9 @@ func (t *transfer) delete(repo *repository) error { return jobStoppedErr } - digests := map[string]struct{}{} repository := repo.repository for _, tag := range repo.tags { - exist, digest, err := t.dst.ManifestExist(repository, tag) + exist, _, err := t.dst.ManifestExist(repository, tag) if err != nil { t.logger.Errorf("failed to check the existence of the manifest of image %s:%s on the destination registry: %v", repository, tag, err) @@ -292,18 +291,12 @@ func (t *transfer) delete(repo *repository) error { repository, tag) continue } - t.logger.Infof("the digest of image %s:%s is %s", repository, tag, digest) - if _, exist := digests[digest]; !exist { - digests[digest] = struct{}{} - } - } - for digest := range digests { - if err := t.dst.DeleteManifest(repository, digest); err != nil { + if err := t.dst.DeleteManifest(repository, tag); err != nil { t.logger.Errorf("failed to delete the manifest of image %s:%s on the destination registry: %v", - repository, digest, err) + repository, tag, err) return err } - t.logger.Infof("the manifest of image %s:%s is deleted", repository, digest) + t.logger.Infof("the manifest of image %s:%s is deleted", repository, tag) } return nil }