From 8768a5678c680d841b33ce208f7af59d814d28f6 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Mon, 1 Jul 2019 18:33:55 +0800 Subject: [PATCH] Merge Default ImageRegistry into the native adapter to reduce the duplicate code Merge Default ImageRegistry into the native adapter to reduce the duplicate code Signed-off-by: Wenkai Yin --- src/replication/adapter/adapter.go | 83 +++++ src/replication/adapter/awsecr/adapter.go | 16 +- .../adapter/awsecr/adapter_test.go | 15 +- .../adapter/awsecr/image_registry.go | 113 ------ src/replication/adapter/azurecr/adapter.go | 19 +- src/replication/adapter/chart_registry.go | 30 -- src/replication/adapter/dockerhub/adapter.go | 11 +- src/replication/adapter/googlegcr/adapter.go | 12 +- .../adapter/googlegcr/image_registry.go | 113 ------ src/replication/adapter/harbor/adapter.go | 7 +- .../adapter/huawei/huawei_adapter.go | 9 +- src/replication/adapter/image_registry.go | 325 ----------------- .../adapter/image_registry_test.go | 46 --- src/replication/adapter/native/adapter.go | 343 ++++++++++++++++-- .../adapter/native/adapter_test.go | 295 ++++++++++++++- .../adapter/native/image_registry.go | 114 ------ .../adapter/native/image_registry_test.go | 279 -------------- 17 files changed, 723 insertions(+), 1107 deletions(-) delete mode 100644 src/replication/adapter/awsecr/image_registry.go delete mode 100644 src/replication/adapter/chart_registry.go delete mode 100644 src/replication/adapter/googlegcr/image_registry.go delete mode 100644 src/replication/adapter/image_registry.go delete mode 100644 src/replication/adapter/image_registry_test.go delete mode 100644 src/replication/adapter/native/image_registry.go delete mode 100644 src/replication/adapter/native/image_registry_test.go diff --git a/src/replication/adapter/adapter.go b/src/replication/adapter/adapter.go index ed502c347..5e944ac7e 100644 --- a/src/replication/adapter/adapter.go +++ b/src/replication/adapter/adapter.go @@ -17,10 +17,18 @@ package adapter import ( "errors" "fmt" + "io" + "github.com/docker/distribution" + "github.com/goharbor/harbor/src/replication/filter" "github.com/goharbor/harbor/src/replication/model" ) +// const definition +const ( + UserAgentReplication = "harbor-replication-service" +) + var registry = map[model.RegistryType]Factory{} // Factory creates a specific Adapter according to the params @@ -37,6 +45,81 @@ type Adapter interface { HealthCheck() (model.HealthStatus, error) } +// ImageRegistry defines the capabilities that an image registry should have +type ImageRegistry interface { + FetchImages(filters []*model.Filter) ([]*model.Resource, error) + ManifestExist(repository, reference string) (exist bool, digest string, err error) + PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error) + PushManifest(repository, reference, mediaType string, payload []byte) error + // the "reference" can be "tag" or "digest", the function needs to handle both + DeleteManifest(repository, reference string) error + BlobExist(repository, digest string) (exist bool, err error) + PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error) + PushBlob(repository, digest string, size int64, blob io.Reader) error +} + +// ChartRegistry defines the capabilities that a chart registry should have +type ChartRegistry interface { + FetchCharts(filters []*model.Filter) ([]*model.Resource, error) + ChartExist(name, version string) (bool, error) + DownloadChart(name, version string) (io.ReadCloser, error) + UploadChart(name, version string, chart io.Reader) error + DeleteChart(name, version string) error +} + +// Repository defines an repository object, it can be image repository, chart repository and etc. +type Repository struct { + ResourceType string `json:"resource_type"` + Name string `json:"name"` +} + +// GetName returns the name +func (r *Repository) GetName() string { + return r.Name +} + +// GetFilterableType returns the filterable type +func (r *Repository) GetFilterableType() filter.FilterableType { + return filter.FilterableTypeRepository +} + +// GetResourceType returns the resource type +func (r *Repository) GetResourceType() string { + return r.ResourceType +} + +// GetLabels returns the labels +func (r *Repository) GetLabels() []string { + return nil +} + +// VTag defines an vTag object, it can be image tag, chart version and etc. +type VTag struct { + ResourceType string `json:"resource_type"` + Name string `json:"name"` + Labels []string `json:"labels"` +} + +// GetFilterableType returns the filterable type +func (v *VTag) GetFilterableType() filter.FilterableType { + return filter.FilterableTypeVTag +} + +// GetResourceType returns the resource type +func (v *VTag) GetResourceType() string { + return v.ResourceType +} + +// GetName returns the name +func (v *VTag) GetName() string { + return v.Name +} + +// GetLabels returns the labels +func (v *VTag) GetLabels() []string { + return v.Labels +} + // RegisterFactory registers one adapter factory to the registry func RegisterFactory(t model.RegistryType, factory Factory) error { if len(t) == 0 { diff --git a/src/replication/adapter/awsecr/adapter.go b/src/replication/adapter/awsecr/adapter.go index 08d327423..2506abc0c 100644 --- a/src/replication/adapter/awsecr/adapter.go +++ b/src/replication/adapter/awsecr/adapter.go @@ -16,6 +16,9 @@ package awsecr import ( "errors" + "net/http" + "regexp" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" @@ -24,9 +27,8 @@ import ( "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/registry" adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/model" - "net/http" - "regexp" ) func init() { @@ -45,14 +47,14 @@ func newAdapter(registry *model.Registry) (*adapter, error) { return nil, err } authorizer := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret, registry.Insecure) - reg, err := adp.NewDefaultImageRegistryWithCustomizedAuthorizer(registry, authorizer) + dockerRegistry, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer) if err != nil { return nil, err } return &adapter{ - registry: registry, - DefaultImageRegistry: reg, - region: region, + registry: registry, + Adapter: dockerRegistry, + region: region, }, nil } @@ -66,7 +68,7 @@ func parseRegion(url string) (string, error) { } type adapter struct { - *adp.DefaultImageRegistry + *native.Adapter registry *model.Registry region string forceEndpoint *string diff --git a/src/replication/adapter/awsecr/adapter_test.go b/src/replication/adapter/awsecr/adapter_test.go index 278aea48b..b473a22ff 100644 --- a/src/replication/adapter/awsecr/adapter_test.go +++ b/src/replication/adapter/awsecr/adapter_test.go @@ -2,8 +2,6 @@ package awsecr import ( "fmt" - "github.com/goharbor/harbor/src/common/utils/test" - "github.com/stretchr/testify/assert" "io" "io/ioutil" "net/http" @@ -11,8 +9,11 @@ import ( "testing" "time" + "github.com/goharbor/harbor/src/common/utils/test" adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/model" + "github.com/stretchr/testify/assert" ) func TestAdapter_NewAdapter(t *testing.T) { @@ -130,15 +131,15 @@ func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Ser AccessSecret: "ppp", } } - reg, err := adp.NewDefaultImageRegistry(registry) + dockerRegistryAdapter, err := native.NewAdapter(registry) if err != nil { panic(err) } return &adapter{ - registry: registry, - DefaultImageRegistry: reg, - region: "test-region", - forceEndpoint: &server.URL, + registry: registry, + Adapter: dockerRegistryAdapter, + region: "test-region", + forceEndpoint: &server.URL, }, server } diff --git a/src/replication/adapter/awsecr/image_registry.go b/src/replication/adapter/awsecr/image_registry.go deleted file mode 100644 index 89d78b76f..000000000 --- a/src/replication/adapter/awsecr/image_registry.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package awsecr - -import ( - adp "github.com/goharbor/harbor/src/replication/adapter" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/util" -) - -var _ adp.ImageRegistry = adapter{} - -func (a adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) { - nameFilterPattern := "" - tagFilterPattern := "" - for _, filter := range filters { - switch filter.Type { - case model.FilterTypeName: - nameFilterPattern = filter.Value.(string) - case model.FilterTypeTag: - tagFilterPattern = filter.Value.(string) - } - } - repositories, err := a.filterRepositories(nameFilterPattern) - if err != nil { - return nil, err - } - - var resources []*model.Resource - for _, repository := range repositories { - tags, err := a.filterTags(repository, tagFilterPattern) - if err != nil { - return nil, err - } - if len(tags) == 0 { - continue - } - resources = append(resources, &model.Resource{ - Type: model.ResourceTypeImage, - Registry: a.registry, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: repository, - }, - Vtags: tags, - }, - }) - } - - return resources, nil -} - -func (a adapter) filterRepositories(pattern string) ([]string, error) { - // if the pattern is a specific repository name, just returns the parsed repositories - // and will check the existence later when filtering the tags - if repositories, ok := util.IsSpecificPath(pattern); ok { - return repositories, nil - } - // search repositories from catalog api - repositories, err := a.Catalog() - if err != nil { - return nil, err - } - // if the pattern is null, just return the result of catalog API - if len(pattern) == 0 { - return repositories, nil - } - result := []string{} - for _, repository := range repositories { - match, err := util.Match(pattern, repository) - if err != nil { - return nil, err - } - if match { - result = append(result, repository) - } - } - return result, nil -} - -func (a adapter) filterTags(repository, pattern string) ([]string, error) { - tags, err := a.ListTag(repository) - if err != nil { - return nil, err - } - if len(pattern) == 0 { - return tags, nil - } - - var result []string - for _, tag := range tags { - match, err := util.Match(pattern, tag) - if err != nil { - return nil, err - } - if match { - result = append(result, tag) - } - } - return result, nil -} diff --git a/src/replication/adapter/azurecr/adapter.go b/src/replication/adapter/azurecr/adapter.go index beaaf24c0..94c0ae46b 100644 --- a/src/replication/adapter/azurecr/adapter.go +++ b/src/replication/adapter/azurecr/adapter.go @@ -24,24 +24,26 @@ func init() { } func factory(registry *model.Registry) (adp.Adapter, error) { - client, err := getClient(registry) - if err != nil { - return nil, err + if registry.Credential == nil || len(registry.Credential.AccessKey) == 0 || + len(registry.Credential.AccessSecret) == 0 { + return nil, fmt.Errorf("credential is necessary for registry %s", registry.URL) } - reg, err := native.NewWithClient(registry, client) + authorizer := auth.NewBasicAuthCredential(registry.Credential.AccessKey, + registry.Credential.AccessSecret) + dockerRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer) if err != nil { return nil, err } return &adapter{ registry: registry, - Native: reg, + Adapter: dockerRegistryAdapter, }, nil } type adapter struct { - *native.Native + *native.Adapter registry *model.Registry } @@ -72,11 +74,6 @@ func (a *adapter) Info() (*model.RegistryInfo, error) { }, nil } -// PrepareForPush no preparation needed for Azure container registry -func (a *adapter) PrepareForPush(resources []*model.Resource) error { - return nil -} - // HealthCheck checks health status of a registry func (a *adapter) HealthCheck() (model.HealthStatus, error) { err := a.PingGet() diff --git a/src/replication/adapter/chart_registry.go b/src/replication/adapter/chart_registry.go deleted file mode 100644 index fef80e18e..000000000 --- a/src/replication/adapter/chart_registry.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package adapter - -import ( - "io" - - "github.com/goharbor/harbor/src/replication/model" -) - -// ChartRegistry defines the capabilities that a chart registry should have -type ChartRegistry interface { - FetchCharts(filters []*model.Filter) ([]*model.Resource, error) - ChartExist(name, version string) (bool, error) - DownloadChart(name, version string) (io.ReadCloser, error) - UploadChart(name, version string, chart io.Reader) error - DeleteChart(name, version string) error -} diff --git a/src/replication/adapter/dockerhub/adapter.go b/src/replication/adapter/dockerhub/adapter.go index 374ebf8c4..86e4d96a2 100644 --- a/src/replication/adapter/dockerhub/adapter.go +++ b/src/replication/adapter/dockerhub/adapter.go @@ -12,6 +12,7 @@ import ( "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/registry/auth" adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/util" ) @@ -47,7 +48,7 @@ func factory(registry *model.Registry) (adp.Adapter, error) { Transport: util.GetHTTPTransport(registry.Insecure), }, credential) - reg, err := adp.NewDefaultImageRegistryWithCustomizedAuthorizer(&model.Registry{ + dockerRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(&model.Registry{ Name: registry.Name, URL: registryURL, // specify the URL of Docker Hub registry service Credential: registry.Credential, @@ -58,14 +59,14 @@ func factory(registry *model.Registry) (adp.Adapter, error) { } return &adapter{ - client: client, - registry: registry, - DefaultImageRegistry: reg, + client: client, + registry: registry, + Adapter: dockerRegistryAdapter, }, nil } type adapter struct { - *adp.DefaultImageRegistry + *native.Adapter registry *model.Registry client *Client } diff --git a/src/replication/adapter/googlegcr/adapter.go b/src/replication/adapter/googlegcr/adapter.go index 6d73da6e1..4e7f40357 100644 --- a/src/replication/adapter/googlegcr/adapter.go +++ b/src/replication/adapter/googlegcr/adapter.go @@ -15,12 +15,14 @@ package googlegcr import ( + "net/http" + "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/registry/auth" adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/util" - "net/http" ) func init() { @@ -44,19 +46,19 @@ func newAdapter(registry *model.Registry) (*adapter, error) { Transport: util.GetHTTPTransport(registry.Insecure), }, credential) - reg, err := adp.NewDefaultImageRegistryWithCustomizedAuthorizer(registry, authorizer) + dockerRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer) if err != nil { return nil, err } return &adapter{ - registry: registry, - DefaultImageRegistry: reg, + registry: registry, + Adapter: dockerRegistryAdapter, }, nil } type adapter struct { - *adp.DefaultImageRegistry + *native.Adapter registry *model.Registry } diff --git a/src/replication/adapter/googlegcr/image_registry.go b/src/replication/adapter/googlegcr/image_registry.go deleted file mode 100644 index 3795bbe98..000000000 --- a/src/replication/adapter/googlegcr/image_registry.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package googlegcr - -import ( - adp "github.com/goharbor/harbor/src/replication/adapter" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/util" -) - -var _ adp.ImageRegistry = adapter{} - -func (a adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) { - nameFilterPattern := "" - tagFilterPattern := "" - for _, filter := range filters { - switch filter.Type { - case model.FilterTypeName: - nameFilterPattern = filter.Value.(string) - case model.FilterTypeTag: - tagFilterPattern = filter.Value.(string) - } - } - repositories, err := a.filterRepositories(nameFilterPattern) - if err != nil { - return nil, err - } - - var resources []*model.Resource - for _, repository := range repositories { - tags, err := a.filterTags(repository, tagFilterPattern) - if err != nil { - return nil, err - } - if len(tags) == 0 { - continue - } - resources = append(resources, &model.Resource{ - Type: model.ResourceTypeImage, - Registry: a.registry, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: repository, - }, - Vtags: tags, - }, - }) - } - - return resources, nil -} - -func (a adapter) filterRepositories(pattern string) ([]string, error) { - // if the pattern is a specific repository name, just returns the parsed repositories - // and will check the existence later when filtering the tags - if repositories, ok := util.IsSpecificPath(pattern); ok { - return repositories, nil - } - // search repositories from catalog api - repositories, err := a.Catalog() - if err != nil { - return nil, err - } - // if the pattern is null, just return the result of catalog API - if len(pattern) == 0 { - return repositories, nil - } - result := []string{} - for _, repository := range repositories { - match, err := util.Match(pattern, repository) - if err != nil { - return nil, err - } - if match { - result = append(result, repository) - } - } - return result, nil -} - -func (a adapter) filterTags(repository, pattern string) ([]string, error) { - tags, err := a.ListTag(repository) - if err != nil { - return nil, err - } - if len(pattern) == 0 { - return tags, nil - } - - var result []string - for _, tag := range tags { - match, err := util.Match(pattern, tag) - if err != nil { - return nil, err - } - if match { - result = append(result, tag) - } - } - return result, nil -} diff --git a/src/replication/adapter/harbor/adapter.go b/src/replication/adapter/harbor/adapter.go index 57ed33045..c24940d61 100644 --- a/src/replication/adapter/harbor/adapter.go +++ b/src/replication/adapter/harbor/adapter.go @@ -27,6 +27,7 @@ import ( "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/common/utils/registry/auth" adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/util" ) @@ -42,7 +43,7 @@ func init() { } type adapter struct { - *adp.DefaultImageRegistry + *native.Adapter registry *model.Registry url string client *common_http.Client @@ -67,7 +68,7 @@ func newAdapter(registry *model.Registry) (*adapter, error) { modifiers = append(modifiers, authorizer) } - reg, err := adp.NewDefaultImageRegistry(registry) + dockerRegistryAdapter, err := native.NewAdapter(registry) if err != nil { return nil, err } @@ -78,7 +79,7 @@ func newAdapter(registry *model.Registry) (*adapter, error) { &http.Client{ Transport: transport, }, modifiers...), - DefaultImageRegistry: reg, + Adapter: dockerRegistryAdapter, }, nil } diff --git a/src/replication/adapter/huawei/huawei_adapter.go b/src/replication/adapter/huawei/huawei_adapter.go index 11dcfed7d..9c3dcb401 100644 --- a/src/replication/adapter/huawei/huawei_adapter.go +++ b/src/replication/adapter/huawei/huawei_adapter.go @@ -12,6 +12,7 @@ import ( "github.com/goharbor/harbor/src/common/utils/log" adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/adapter/native" "github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/util" ) @@ -27,7 +28,7 @@ func init() { // Adapter is for images replications between harbor and Huawei image repository(SWR) type adapter struct { - *adp.DefaultImageRegistry + *native.Adapter registry *model.Registry } @@ -232,13 +233,13 @@ func (a *adapter) HealthCheck() (model.HealthStatus, error) { // AdapterFactory is the factory for huawei adapter func AdapterFactory(registry *model.Registry) (adp.Adapter, error) { - reg, err := adp.NewDefaultImageRegistry(registry) + dockerRegistryAdapter, err := native.NewAdapter(registry) if err != nil { return nil, err } return &adapter{ - registry: registry, - DefaultImageRegistry: reg, + registry: registry, + Adapter: dockerRegistryAdapter, }, nil } diff --git a/src/replication/adapter/image_registry.go b/src/replication/adapter/image_registry.go deleted file mode 100644 index 02bb10266..000000000 --- a/src/replication/adapter/image_registry.go +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package adapter - -import ( - "errors" - "io" - "net/http" - "strings" - "sync" - - "github.com/goharbor/harbor/src/replication/filter" - - "github.com/docker/distribution" - "github.com/docker/distribution/manifest/schema1" - "github.com/goharbor/harbor/src/common/http/modifier" - common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth" - "github.com/goharbor/harbor/src/common/utils/log" - registry_pkg "github.com/goharbor/harbor/src/common/utils/registry" - "github.com/goharbor/harbor/src/common/utils/registry/auth" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/util" -) - -// const definition -const ( - UserAgentReplication = "harbor-replication-service" -) - -// ImageRegistry defines the capabilities that an image registry should have -type ImageRegistry interface { - FetchImages(filters []*model.Filter) ([]*model.Resource, error) - ManifestExist(repository, reference string) (exist bool, digest string, err error) - PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error) - PushManifest(repository, reference, mediaType string, payload []byte) error - // the "reference" can be "tag" or "digest", the function needs to handle both - DeleteManifest(repository, reference string) error - BlobExist(repository, digest string) (exist bool, err error) - PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error) - PushBlob(repository, digest string, size int64, blob io.Reader) error -} - -// Repository defines an repository object, it can be image repository, chart repository and etc. -type Repository struct { - ResourceType string `json:"resource_type"` - Name string `json:"name"` -} - -// GetName returns the name -func (r *Repository) GetName() string { - return r.Name -} - -// GetFilterableType returns the filterable type -func (r *Repository) GetFilterableType() filter.FilterableType { - return filter.FilterableTypeRepository -} - -// GetResourceType returns the resource type -func (r *Repository) GetResourceType() string { - return r.ResourceType -} - -// GetLabels returns the labels -func (r *Repository) GetLabels() []string { - return nil -} - -// VTag defines an vTag object, it can be image tag, chart version and etc. -type VTag struct { - ResourceType string `json:"resource_type"` - Name string `json:"name"` - Labels []string `json:"labels"` -} - -// GetFilterableType returns the filterable type -func (v *VTag) GetFilterableType() filter.FilterableType { - return filter.FilterableTypeVTag -} - -// GetResourceType returns the resource type -func (v *VTag) GetResourceType() string { - return v.ResourceType -} - -// GetName returns the name -func (v *VTag) GetName() string { - return v.Name -} - -// GetLabels returns the labels -func (v *VTag) GetLabels() []string { - return v.Labels -} - -// DefaultImageRegistry provides a default implementation for interface ImageRegistry -type DefaultImageRegistry struct { - sync.RWMutex - *registry_pkg.Registry - registry *model.Registry - client *http.Client - clients map[string]*registry_pkg.Repository -} - -// NewDefaultRegistryWithClient returns an instance of DefaultImageRegistry -func NewDefaultRegistryWithClient(registry *model.Registry, client *http.Client) (*DefaultImageRegistry, error) { - reg, err := registry_pkg.NewRegistry(registry.URL, client) - if err != nil { - return nil, err - } - - return &DefaultImageRegistry{ - Registry: reg, - client: client, - registry: registry, - clients: map[string]*registry_pkg.Repository{}, - }, nil -} - -// NewDefaultImageRegistry returns an instance of DefaultImageRegistry -func NewDefaultImageRegistry(registry *model.Registry) (*DefaultImageRegistry, error) { - var authorizer modifier.Modifier - if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 { - var cred modifier.Modifier - if registry.Credential.Type == model.CredentialTypeSecret { - cred = common_http_auth.NewSecretAuthorizer(registry.Credential.AccessSecret) - } else { - cred = auth.NewBasicAuthCredential( - registry.Credential.AccessKey, - registry.Credential.AccessSecret) - } - authorizer = auth.NewStandardTokenAuthorizer(&http.Client{ - Transport: util.GetHTTPTransport(registry.Insecure), - }, cred, registry.TokenServiceURL) - } - return NewDefaultImageRegistryWithCustomizedAuthorizer(registry, authorizer) -} - -// NewDefaultImageRegistryWithCustomizedAuthorizer returns an instance of DefaultImageRegistry with the customized authorizer -func NewDefaultImageRegistryWithCustomizedAuthorizer(registry *model.Registry, authorizer modifier.Modifier) (*DefaultImageRegistry, error) { - transport := util.GetHTTPTransport(registry.Insecure) - modifiers := []modifier.Modifier{ - &auth.UserAgentModifier{ - UserAgent: UserAgentReplication, - }, - } - if authorizer != nil { - modifiers = append(modifiers, authorizer) - } - client := &http.Client{ - Transport: registry_pkg.NewTransport(transport, modifiers...), - } - reg, err := registry_pkg.NewRegistry(registry.URL, client) - if err != nil { - return nil, err - } - return &DefaultImageRegistry{ - Registry: reg, - client: client, - registry: registry, - clients: map[string]*registry_pkg.Repository{}, - }, nil -} - -func (d *DefaultImageRegistry) getClient(repository string) (*registry_pkg.Repository, error) { - d.RLock() - client, exist := d.clients[repository] - d.RUnlock() - if exist { - return client, nil - } - - return d.create(repository) -} - -func (d *DefaultImageRegistry) create(repository string) (*registry_pkg.Repository, error) { - d.Lock() - defer d.Unlock() - // double check - client, exist := d.clients[repository] - if exist { - return client, nil - } - - client, err := registry_pkg.NewRepository(repository, d.registry.URL, d.client) - if err != nil { - return nil, err - } - d.clients[repository] = client - return client, nil -} - -// HealthCheck checks health status of a registry -func (d *DefaultImageRegistry) HealthCheck() (model.HealthStatus, error) { - var err error - if d.registry.Credential == nil || - (len(d.registry.Credential.AccessKey) == 0 && len(d.registry.Credential.AccessSecret) == 0) { - err = d.PingSimple() - } else { - err = d.Ping() - } - if err != nil { - log.Errorf("failed to ping registry %s: %v", d.registry.URL, err) - return model.Unhealthy, nil - } - return model.Healthy, nil -} - -// FetchImages ... -func (d *DefaultImageRegistry) FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) { - return nil, errors.New("not implemented") -} - -// ManifestExist ... -func (d *DefaultImageRegistry) ManifestExist(repository, reference string) (bool, string, error) { - client, err := d.getClient(repository) - if err != nil { - return false, "", err - } - digest, exist, err := client.ManifestExist(reference) - return exist, digest, err -} - -// PullManifest ... -func (d *DefaultImageRegistry) PullManifest(repository, reference string, accepttedMediaTypes []string) (distribution.Manifest, string, error) { - client, err := d.getClient(repository) - if err != nil { - return nil, "", err - } - digest, mediaType, payload, err := client.PullManifest(reference, accepttedMediaTypes) - if err != nil { - return nil, "", err - } - if strings.Contains(mediaType, "application/json") { - mediaType = schema1.MediaTypeManifest - } - manifest, _, err := registry_pkg.UnMarshal(mediaType, payload) - if err != nil { - return nil, "", err - } - return manifest, digest, nil -} - -// PushManifest ... -func (d *DefaultImageRegistry) PushManifest(repository, reference, mediaType string, payload []byte) error { - client, err := d.getClient(repository) - if err != nil { - return err - } - _, err = client.PushManifest(reference, mediaType, payload) - return err -} - -// DeleteManifest ... -func (d *DefaultImageRegistry) DeleteManifest(repository, reference string) error { - client, err := d.getClient(repository) - if err != nil { - return err - } - digest := reference - if !isDigest(digest) { - dgt, exist, err := client.ManifestExist(reference) - if err != nil { - return err - } - if !exist { - log.Debugf("the manifest of %s:%s doesn't exist", repository, reference) - return nil - } - digest = dgt - } - return client.DeleteManifest(digest) -} - -// BlobExist ... -func (d *DefaultImageRegistry) BlobExist(repository, digest string) (bool, error) { - client, err := d.getClient(repository) - if err != nil { - return false, err - } - return client.BlobExist(digest) -} - -// PullBlob ... -func (d *DefaultImageRegistry) PullBlob(repository, digest string) (int64, io.ReadCloser, error) { - client, err := d.getClient(repository) - if err != nil { - return 0, nil, err - } - return client.PullBlob(digest) -} - -// PushBlob ... -func (d *DefaultImageRegistry) PushBlob(repository, digest string, size int64, blob io.Reader) error { - client, err := d.getClient(repository) - if err != nil { - return err - } - return client.PushBlob(digest, size, blob) -} - -func isDigest(str string) bool { - return strings.Contains(str, ":") -} - -// ListTag ... -func (d *DefaultImageRegistry) ListTag(repository string) ([]string, error) { - client, err := d.getClient(repository) - if err != nil { - return []string{}, err - } - return client.ListTag() -} diff --git a/src/replication/adapter/image_registry_test.go b/src/replication/adapter/image_registry_test.go deleted file mode 100644 index 157471d41..000000000 --- a/src/replication/adapter/image_registry_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package adapter - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -// TODO add UT - -func TestIsDigest(t *testing.T) { - cases := []struct { - str string - isDigest bool - }{ - { - str: "", - isDigest: false, - }, - { - str: "latest", - isDigest: false, - }, - { - str: "sha256:fea8895f450959fa676bcc1df0611ea93823a735a01205fd8622846041d0c7cf", - isDigest: true, - }, - } - for _, c := range cases { - assert.Equal(t, c.isDigest, isDigest(c.str)) - } -} diff --git a/src/replication/adapter/native/adapter.go b/src/replication/adapter/native/adapter.go index 2793d4d4f..46a8731d6 100644 --- a/src/replication/adapter/native/adapter.go +++ b/src/replication/adapter/native/adapter.go @@ -15,16 +15,26 @@ package native import ( + "io" "net/http" + "strings" + "sync" + "github.com/docker/distribution" + "github.com/docker/distribution/manifest/schema1" + "github.com/goharbor/harbor/src/common/http/modifier" + common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth" "github.com/goharbor/harbor/src/common/utils/log" + registry_pkg "github.com/goharbor/harbor/src/common/utils/registry" + "github.com/goharbor/harbor/src/common/utils/registry/auth" adp "github.com/goharbor/harbor/src/replication/adapter" "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/replication/util" ) func init() { if err := adp.RegisterFactory(model.RegistryTypeDockerRegistry, func(registry *model.Registry) (adp.Adapter, error) { - return newAdapter(registry) + return NewAdapter(registry) }); err != nil { log.Errorf("failed to register factory for %s: %v", model.RegistryTypeDockerRegistry, err) return @@ -32,39 +42,65 @@ func init() { log.Infof("the factory for adapter %s registered", model.RegistryTypeDockerRegistry) } -func newAdapter(registry *model.Registry) (*Native, error) { - reg, err := adp.NewDefaultImageRegistry(registry) - if err != nil { - return nil, err - } - return &Native{ - registry: registry, - DefaultImageRegistry: reg, - }, nil -} +var _ adp.Adapter = &Adapter{} -// NewWithClient ... -func NewWithClient(registry *model.Registry, client *http.Client) (*Native, error) { - reg, err := adp.NewDefaultRegistryWithClient(registry, client) - if err != nil { - return nil, err - } - return &Native{ - registry: registry, - DefaultImageRegistry: reg, - }, nil -} - -// Native is adapter to native docker registry -type Native struct { - *adp.DefaultImageRegistry +// Adapter implements an adapter for Docker registry. It can be used to all registries +// that implement the registry V2 API +type Adapter struct { + sync.RWMutex + *registry_pkg.Registry registry *model.Registry + client *http.Client + clients map[string]*registry_pkg.Repository // client for repositories } -var _ adp.Adapter = Native{} +// NewAdapter returns an instance of the Adapter +func NewAdapter(registry *model.Registry) (*Adapter, error) { + var authorizer modifier.Modifier + if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 { + var cred modifier.Modifier + if registry.Credential.Type == model.CredentialTypeSecret { + cred = common_http_auth.NewSecretAuthorizer(registry.Credential.AccessSecret) + } else { + cred = auth.NewBasicAuthCredential( + registry.Credential.AccessKey, + registry.Credential.AccessSecret) + } + authorizer = auth.NewStandardTokenAuthorizer(&http.Client{ + Transport: util.GetHTTPTransport(registry.Insecure), + }, cred, registry.TokenServiceURL) + } + return NewAdapterWithCustomizedAuthorizer(registry, authorizer) +} -// Info ... -func (Native) Info() (info *model.RegistryInfo, err error) { +// NewAdapterWithCustomizedAuthorizer returns an instance of the Adapter with the customized authorizer +func NewAdapterWithCustomizedAuthorizer(registry *model.Registry, authorizer modifier.Modifier) (*Adapter, error) { + transport := util.GetHTTPTransport(registry.Insecure) + modifiers := []modifier.Modifier{ + &auth.UserAgentModifier{ + UserAgent: adp.UserAgentReplication, + }, + } + if authorizer != nil { + modifiers = append(modifiers, authorizer) + } + client := &http.Client{ + Transport: registry_pkg.NewTransport(transport, modifiers...), + } + reg, err := registry_pkg.NewRegistry(registry.URL, client) + if err != nil { + return nil, err + } + return &Adapter{ + Registry: reg, + registry: registry, + client: client, + clients: map[string]*registry_pkg.Repository{}, + }, nil +} + +// Info returns the basic information about the adapter +func (a *Adapter) Info() (info *model.RegistryInfo, err error) { return &model.RegistryInfo{ Type: model.RegistryTypeDockerRegistry, SupportedResourceTypes: []model.ResourceType{ @@ -87,5 +123,250 @@ func (Native) Info() (info *model.RegistryInfo, err error) { }, nil } -// PrepareForPush nothing need to do. -func (Native) PrepareForPush([]*model.Resource) error { return nil } +// PrepareForPush does nothing +func (a *Adapter) PrepareForPush([]*model.Resource) error { + return nil +} + +// HealthCheck checks health status of a registry +func (a *Adapter) HealthCheck() (model.HealthStatus, error) { + var err error + if a.registry.Credential == nil || + (len(a.registry.Credential.AccessKey) == 0 && len(a.registry.Credential.AccessSecret) == 0) { + err = a.PingSimple() + } else { + err = a.Ping() + } + if err != nil { + log.Errorf("failed to ping registry %s: %v", a.registry.URL, err) + return model.Unhealthy, nil + } + return model.Healthy, nil +} + +// FetchImages ... +func (a *Adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) { + repositories, err := a.getRepositories(filters) + if err != nil { + return nil, err + } + if len(repositories) == 0 { + return nil, nil + } + for _, filter := range filters { + if err = filter.DoFilter(&repositories); err != nil { + return nil, err + } + } + + var resources []*model.Resource + for _, repository := range repositories { + vTags, err := a.getVTags(repository.Name) + if err != nil { + return nil, err + } + if len(vTags) == 0 { + continue + } + for _, filter := range filters { + if err = filter.DoFilter(&vTags); err != nil { + return nil, err + } + } + if len(vTags) == 0 { + continue + } + tags := []string{} + for _, vTag := range vTags { + tags = append(tags, vTag.Name) + } + resources = append(resources, &model.Resource{ + Type: model.ResourceTypeImage, + Registry: a.registry, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: repository.Name, + }, + Vtags: tags, + }, + }) + } + + return resources, nil +} + +func (a *Adapter) getRepositories(filters []*model.Filter) ([]*adp.Repository, error) { + pattern := "" + for _, filter := range filters { + if filter.Type == model.FilterTypeName { + pattern = filter.Value.(string) + break + } + } + var repositories []string + var err error + // if the pattern of repository name filter is a specific repository name, just returns + // the parsed repositories and will check the existence later when filtering the tags + if paths, ok := util.IsSpecificPath(pattern); ok { + repositories = paths + } else { + // search repositories from catalog API + repositories, err = a.Catalog() + if err != nil { + return nil, err + } + } + + result := []*adp.Repository{} + for _, repository := range repositories { + result = append(result, &adp.Repository{ + ResourceType: string(model.ResourceTypeImage), + Name: repository, + }) + } + return result, nil +} + +func (a *Adapter) getVTags(repository string) ([]*adp.VTag, error) { + tags, err := a.ListTag(repository) + if err != nil { + return nil, err + } + var result []*adp.VTag + for _, tag := range tags { + result = append(result, &adp.VTag{ + ResourceType: string(model.ResourceTypeImage), + Name: tag, + }) + } + return result, nil +} + +// ManifestExist ... +func (a *Adapter) ManifestExist(repository, reference string) (bool, string, error) { + client, err := a.getClient(repository) + if err != nil { + return false, "", err + } + digest, exist, err := client.ManifestExist(reference) + return exist, digest, err +} + +// PullManifest ... +func (a *Adapter) PullManifest(repository, reference string, accepttedMediaTypes []string) (distribution.Manifest, string, error) { + client, err := a.getClient(repository) + if err != nil { + return nil, "", err + } + digest, mediaType, payload, err := client.PullManifest(reference, accepttedMediaTypes) + if err != nil { + return nil, "", err + } + if strings.Contains(mediaType, "application/json") { + mediaType = schema1.MediaTypeManifest + } + manifest, _, err := registry_pkg.UnMarshal(mediaType, payload) + if err != nil { + return nil, "", err + } + return manifest, digest, nil +} + +// PushManifest ... +func (a *Adapter) PushManifest(repository, reference, mediaType string, payload []byte) error { + client, err := a.getClient(repository) + if err != nil { + return err + } + _, err = client.PushManifest(reference, mediaType, payload) + return err +} + +// DeleteManifest ... +func (a *Adapter) DeleteManifest(repository, reference string) error { + client, err := a.getClient(repository) + if err != nil { + return err + } + digest := reference + if !isDigest(digest) { + dgt, exist, err := client.ManifestExist(reference) + if err != nil { + return err + } + if !exist { + log.Debugf("the manifest of %s:%s doesn't exist", repository, reference) + return nil + } + digest = dgt + } + return client.DeleteManifest(digest) +} + +// BlobExist ... +func (a *Adapter) BlobExist(repository, digest string) (bool, error) { + client, err := a.getClient(repository) + if err != nil { + return false, err + } + return client.BlobExist(digest) +} + +// PullBlob ... +func (a *Adapter) PullBlob(repository, digest string) (int64, io.ReadCloser, error) { + client, err := a.getClient(repository) + if err != nil { + return 0, nil, err + } + return client.PullBlob(digest) +} + +// PushBlob ... +func (a *Adapter) PushBlob(repository, digest string, size int64, blob io.Reader) error { + client, err := a.getClient(repository) + if err != nil { + return err + } + return client.PushBlob(digest, size, blob) +} + +func isDigest(str string) bool { + return strings.Contains(str, ":") +} + +// ListTag ... +func (a *Adapter) ListTag(repository string) ([]string, error) { + client, err := a.getClient(repository) + if err != nil { + return []string{}, err + } + return client.ListTag() +} + +func (a *Adapter) getClient(repository string) (*registry_pkg.Repository, error) { + a.RLock() + client, exist := a.clients[repository] + a.RUnlock() + if exist { + return client, nil + } + + return a.create(repository) +} + +func (a *Adapter) create(repository string) (*registry_pkg.Repository, error) { + a.Lock() + defer a.Unlock() + // double check + client, exist := a.clients[repository] + if exist { + return client, nil + } + + client, err := registry_pkg.NewRepository(repository, a.registry.URL, a.client) + if err != nil { + return nil, err + } + a.clients[repository] = client + return client, nil +} diff --git a/src/replication/adapter/native/adapter_test.go b/src/replication/adapter/native/adapter_test.go index 43d993fa0..27b0fe4f0 100644 --- a/src/replication/adapter/native/adapter_test.go +++ b/src/replication/adapter/native/adapter_test.go @@ -15,11 +15,15 @@ package native import ( + "fmt" + "net/http" + "net/http/httptest" "testing" - adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/common/utils/test" "github.com/goharbor/harbor/src/replication/model" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_newAdapter(t *testing.T) { @@ -33,7 +37,7 @@ func Test_newAdapter(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := newAdapter(tt.registry) + got, err := NewAdapter(tt.registry) if tt.wantErr { assert.NotNil(t, err) assert.Nil(t, got) @@ -47,14 +51,11 @@ func Test_newAdapter(t *testing.T) { func Test_native_Info(t *testing.T) { var registry = &model.Registry{URL: "abc"} - var reg, _ = adp.NewDefaultImageRegistry(registry) - var adapter = Native{ - DefaultImageRegistry: reg, - registry: registry, - } + adapter, err := NewAdapter(registry) + require.Nil(t, err) assert.NotNil(t, adapter) - var info, err = adapter.Info() + info, err := adapter.Info() assert.Nil(t, err) assert.NotNil(t, info) assert.Equal(t, model.RegistryTypeDockerRegistry, info.Type) @@ -66,13 +67,279 @@ func Test_native_Info(t *testing.T) { func Test_native_PrepareForPush(t *testing.T) { var registry = &model.Registry{URL: "abc"} - var reg, _ = adp.NewDefaultImageRegistry(registry) - var adapter = Native{ - DefaultImageRegistry: reg, - registry: registry, - } + adapter, err := NewAdapter(registry) + require.Nil(t, err) assert.NotNil(t, adapter) - var err = adapter.PrepareForPush(nil) + err = adapter.PrepareForPush(nil) assert.Nil(t, err) } + +func mockNativeRegistry() (mock *httptest.Server) { + return test.NewServer( + &test.RequestHandlerMapping{ + Method: http.MethodGet, + Pattern: "/v2/_catalog", + Handler: func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"repositories":["test/a1","test/b2","test/c3/3level"]}`)) + }, + }, + &test.RequestHandlerMapping{ + Method: http.MethodGet, + Pattern: "/v2/test/a1/tags/list", + Handler: func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"name":"test/a1","tags":["tag11"]}`)) + }, + }, + &test.RequestHandlerMapping{ + Method: http.MethodGet, + Pattern: "/v2/test/b2/tags/list", + Handler: func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"name":"test/b2","tags":["tag11","tag2","tag13"]}`)) + }, + }, + &test.RequestHandlerMapping{ + Method: http.MethodGet, + Pattern: "/v2/test/c3/3level/tags/list", + Handler: func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"name":"test/c3/3level","tags":["tag4"]}`)) + }, + }, + ) +} +func Test_native_FetchImages(t *testing.T) { + var mock = mockNativeRegistry() + defer mock.Close() + fmt.Println("mockNativeRegistry URL: ", mock.URL) + + var registry = &model.Registry{ + Type: model.RegistryTypeDockerRegistry, + URL: mock.URL, + Insecure: true, + } + adapter, err := NewAdapter(registry) + assert.Nil(t, err) + assert.NotNil(t, adapter) + + tests := []struct { + name string + filters []*model.Filter + want []*model.Resource + wantErr bool + }{ + { + name: "repository not exist", + filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "b1", + }, + }, + wantErr: false, + }, + { + name: "tag not exist", + filters: []*model.Filter{ + { + Type: model.FilterTypeTag, + Value: "this_tag_not_exist_in_the_mock_server", + }, + }, + wantErr: false, + }, + { + name: "no filters", + filters: []*model.Filter{}, + want: []*model.Resource{ + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{Name: "test/a1"}, + Vtags: []string{"tag11"}, + }, + }, + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{Name: "test/b2"}, + Vtags: []string{"tag11", "tag2", "tag13"}, + }, + }, + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{Name: "test/c3/3level"}, + Vtags: []string{"tag4"}, + }, + }, + }, + wantErr: false, + }, + { + name: "only special repository", + filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "test/a1", + }, + }, + want: []*model.Resource{ + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{Name: "test/a1"}, + Vtags: []string{"tag11"}, + }, + }, + }, + wantErr: false, + }, + { + name: "only special tag", + filters: []*model.Filter{ + { + Type: model.FilterTypeTag, + Value: "tag11", + }, + }, + want: []*model.Resource{ + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{Name: "test/a1"}, + Vtags: []string{"tag11"}, + }, + }, + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{Name: "test/b2"}, + Vtags: []string{"tag11"}, + }, + }, + }, + wantErr: false, + }, + { + name: "special repository and special tag", + filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "test/b2", + }, + { + Type: model.FilterTypeTag, + Value: "tag2", + }, + }, + want: []*model.Resource{ + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{Name: "test/b2"}, + Vtags: []string{"tag2"}, + }, + }, + }, + + wantErr: false, + }, + { + name: "only wildcard repository", + filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "test/b*", + }, + }, + want: []*model.Resource{ + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{Name: "test/b2"}, + Vtags: []string{"tag11", "tag2", "tag13"}, + }, + }, + }, + wantErr: false, + }, + { + name: "only wildcard tag", + filters: []*model.Filter{ + { + Type: model.FilterTypeTag, + Value: "tag1*", + }, + }, + want: []*model.Resource{ + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{Name: "test/a1"}, + Vtags: []string{"tag11"}, + }, + }, + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{Name: "test/b2"}, + Vtags: []string{"tag11", "tag13"}, + }, + }, + }, + wantErr: false, + }, + { + name: "wildcard repository and wildcard tag", + filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "test/b*", + }, + { + Type: model.FilterTypeTag, + Value: "tag1*", + }, + }, + want: []*model.Resource{ + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{Name: "test/b2"}, + Vtags: []string{"tag11", "tag13"}, + }, + }, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var resources, err = adapter.FetchImages(tt.filters) + if tt.wantErr { + require.Len(t, resources, 0) + require.NotNil(t, err) + } else { + require.Equal(t, len(tt.want), len(resources)) + for i, resource := range resources { + require.NotNil(t, resource.Metadata) + assert.Equal(t, tt.want[i].Metadata.Repository, resource.Metadata.Repository) + assert.Equal(t, tt.want[i].Metadata.Vtags, resource.Metadata.Vtags) + } + } + }) + } +} + +func TestIsDigest(t *testing.T) { + cases := []struct { + str string + isDigest bool + }{ + { + str: "", + isDigest: false, + }, + { + str: "latest", + isDigest: false, + }, + { + str: "sha256:fea8895f450959fa676bcc1df0611ea93823a735a01205fd8622846041d0c7cf", + isDigest: true, + }, + } + for _, c := range cases { + assert.Equal(t, c.isDigest, isDigest(c.str)) + } +} diff --git a/src/replication/adapter/native/image_registry.go b/src/replication/adapter/native/image_registry.go deleted file mode 100644 index 9f3f387be..000000000 --- a/src/replication/adapter/native/image_registry.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package native - -import ( - adp "github.com/goharbor/harbor/src/replication/adapter" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/util" -) - -var _ adp.ImageRegistry = Native{} - -// FetchImages ... -func (n Native) FetchImages(filters []*model.Filter) ([]*model.Resource, error) { - nameFilterPattern := "" - tagFilterPattern := "" - for _, filter := range filters { - switch filter.Type { - case model.FilterTypeName: - nameFilterPattern = filter.Value.(string) - case model.FilterTypeTag: - tagFilterPattern = filter.Value.(string) - } - } - repositories, err := n.filterRepositories(nameFilterPattern) - if err != nil { - return nil, err - } - - var resources []*model.Resource - for _, repository := range repositories { - tags, err := n.filterTags(repository, tagFilterPattern) - if err != nil { - return nil, err - } - if len(tags) == 0 { - continue - } - resources = append(resources, &model.Resource{ - Type: model.ResourceTypeImage, - Registry: n.registry, - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: repository, - }, - Vtags: tags, - }, - }) - } - - return resources, nil -} - -func (n Native) filterRepositories(pattern string) ([]string, error) { - // if the pattern is a specific repository name, just returns the parsed repositories - // and will check the existence later when filtering the tags - if repositories, ok := util.IsSpecificPath(pattern); ok { - return repositories, nil - } - // search repositories from catalog api - repositories, err := n.Catalog() - if err != nil { - return nil, err - } - // if the pattern is null, just return the result of catalog API - if len(pattern) == 0 { - return repositories, nil - } - result := []string{} - for _, repository := range repositories { - match, err := util.Match(pattern, repository) - if err != nil { - return nil, err - } - if match { - result = append(result, repository) - } - } - return result, nil -} - -func (n Native) filterTags(repository, pattern string) ([]string, error) { - tags, err := n.ListTag(repository) - if err != nil { - return nil, err - } - if len(pattern) == 0 { - return tags, nil - } - - var result []string - for _, tag := range tags { - match, err := util.Match(pattern, tag) - if err != nil { - return nil, err - } - if match { - result = append(result, tag) - } - } - return result, nil -} diff --git a/src/replication/adapter/native/image_registry_test.go b/src/replication/adapter/native/image_registry_test.go deleted file mode 100644 index abd2baba7..000000000 --- a/src/replication/adapter/native/image_registry_test.go +++ /dev/null @@ -1,279 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package native - -import ( - "fmt" - "net/http" - "net/http/httptest" - "testing" - - "github.com/goharbor/harbor/src/common/utils/test" - adp "github.com/goharbor/harbor/src/replication/adapter" - "github.com/goharbor/harbor/src/replication/model" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func mockNativeRegistry() (mock *httptest.Server) { - return test.NewServer( - &test.RequestHandlerMapping{ - Method: http.MethodGet, - Pattern: "/v2/_catalog", - Handler: func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"repositories":["test/a1","test/b2","test/c3/3level"]}`)) - }, - }, - &test.RequestHandlerMapping{ - Method: http.MethodGet, - Pattern: "/v2/test/a1/tags/list", - Handler: func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"name":"test/a1","tags":["tag11"]}`)) - }, - }, - &test.RequestHandlerMapping{ - Method: http.MethodGet, - Pattern: "/v2/test/b2/tags/list", - Handler: func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"name":"test/b2","tags":["tag11","tag2","tag13"]}`)) - }, - }, - &test.RequestHandlerMapping{ - Method: http.MethodGet, - Pattern: "/v2/test/c3/3level/tags/list", - Handler: func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"name":"test/c3/3level","tags":["tag4"]}`)) - }, - }, - ) -} -func Test_native_FetchImages(t *testing.T) { - var mock = mockNativeRegistry() - defer mock.Close() - fmt.Println("mockNativeRegistry URL: ", mock.URL) - - var registry = &model.Registry{ - Type: model.RegistryTypeDockerRegistry, - URL: mock.URL, - Insecure: true, - } - var reg, err = adp.NewDefaultImageRegistry(registry) - assert.NotNil(t, reg) - assert.Nil(t, err) - var adapter = Native{ - DefaultImageRegistry: reg, - registry: registry, - } - assert.NotNil(t, adapter) - - tests := []struct { - name string - filters []*model.Filter - want []*model.Resource - wantErr bool - }{ - { - name: "repository not exist", - filters: []*model.Filter{ - { - Type: model.FilterTypeName, - Value: "b1", - }, - }, - wantErr: false, - }, - { - name: "tag not exist", - filters: []*model.Filter{ - { - Type: model.FilterTypeTag, - Value: "this_tag_not_exist_in_the_mock_server", - }, - }, - wantErr: false, - }, - { - name: "no filters", - filters: []*model.Filter{}, - want: []*model.Resource{ - { - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{Name: "test/a1"}, - Vtags: []string{"tag11"}, - }, - }, - { - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{Name: "test/b2"}, - Vtags: []string{"tag11", "tag2", "tag13"}, - }, - }, - { - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{Name: "test/c3/3level"}, - Vtags: []string{"tag4"}, - }, - }, - }, - wantErr: false, - }, - { - name: "only special repository", - filters: []*model.Filter{ - { - Type: model.FilterTypeName, - Value: "test/a1", - }, - }, - want: []*model.Resource{ - { - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{Name: "test/a1"}, - Vtags: []string{"tag11"}, - }, - }, - }, - wantErr: false, - }, - { - name: "only special tag", - filters: []*model.Filter{ - { - Type: model.FilterTypeTag, - Value: "tag11", - }, - }, - want: []*model.Resource{ - { - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{Name: "test/a1"}, - Vtags: []string{"tag11"}, - }, - }, - { - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{Name: "test/b2"}, - Vtags: []string{"tag11"}, - }, - }, - }, - wantErr: false, - }, - { - name: "special repository and special tag", - filters: []*model.Filter{ - { - Type: model.FilterTypeName, - Value: "test/b2", - }, - { - Type: model.FilterTypeTag, - Value: "tag2", - }, - }, - want: []*model.Resource{ - { - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{Name: "test/b2"}, - Vtags: []string{"tag2"}, - }, - }, - }, - - wantErr: false, - }, - { - name: "only wildcard repository", - filters: []*model.Filter{ - { - Type: model.FilterTypeName, - Value: "test/b*", - }, - }, - want: []*model.Resource{ - { - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{Name: "test/b2"}, - Vtags: []string{"tag11", "tag2", "tag13"}, - }, - }, - }, - wantErr: false, - }, - { - name: "only wildcard tag", - filters: []*model.Filter{ - { - Type: model.FilterTypeTag, - Value: "tag1*", - }, - }, - want: []*model.Resource{ - { - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{Name: "test/a1"}, - Vtags: []string{"tag11"}, - }, - }, - { - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{Name: "test/b2"}, - Vtags: []string{"tag11", "tag13"}, - }, - }, - }, - wantErr: false, - }, - { - name: "wildcard repository and wildcard tag", - filters: []*model.Filter{ - { - Type: model.FilterTypeName, - Value: "test/b*", - }, - { - Type: model.FilterTypeTag, - Value: "tag1*", - }, - }, - want: []*model.Resource{ - { - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{Name: "test/b2"}, - Vtags: []string{"tag11", "tag13"}, - }, - }, - }, - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var resources, err = adapter.FetchImages(tt.filters) - if tt.wantErr { - require.Len(t, resources, 0) - require.NotNil(t, err) - } else { - require.Equal(t, len(tt.want), len(resources)) - for i, resource := range resources { - require.NotNil(t, resource.Metadata) - assert.Equal(t, tt.want[i].Metadata.Repository, resource.Metadata.Repository) - assert.Equal(t, tt.want[i].Metadata.Vtags, resource.Metadata.Vtags) - } - } - }) - } -}