From 5a047a7eb61aabf2bd64caeb842ac1d2ab223b7e Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 10 Apr 2019 19:37:33 +0800 Subject: [PATCH] Update the adapter interface Add ConvertResourceMetadata and PrepareForPush methods Signed-off-by: Wenkai Yin --- src/core/api/repository.go | 11 +- .../service/notifications/registry/handler.go | 11 +- src/replication/ng/adapter/adapter.go | 11 +- src/replication/ng/adapter/harbor/adapter.go | 65 +++++++- .../ng/adapter/harbor/adapter_test.go | 53 +++++-- .../ng/adapter/harbor/chart_registry.go | 11 +- .../ng/adapter/harbor/chart_registry_test.go | 4 +- .../ng/adapter/harbor/image_registry.go | 12 +- .../ng/adapter/harbor/image_registry_test.go | 4 +- src/replication/ng/event/handler.go | 4 +- src/replication/ng/event/handler_test.go | 40 +++-- src/replication/ng/model/registry.go | 1 + src/replication/ng/model/resource.go | 30 +++- src/replication/ng/model/resource_test.go | 50 ++++++ src/replication/ng/operation/controller.go | 5 +- .../ng/operation/controller_test.go | 40 ++++- src/replication/ng/operation/flow/copy.go | 5 +- src/replication/ng/operation/flow/deletion.go | 9 +- .../ng/operation/flow/deletion_test.go | 10 +- src/replication/ng/operation/flow/stage.go | 96 +++++------- .../ng/operation/flow/stage_test.go | 144 +++++++++--------- .../ng/operation/scheduler/scheduler_test.go | 16 +- src/replication/ng/transfer/chart/transfer.go | 9 +- .../ng/transfer/chart/transfer_test.go | 10 +- .../ng/transfer/repository/transfer.go | 9 +- 25 files changed, 441 insertions(+), 219 deletions(-) create mode 100644 src/replication/ng/model/resource_test.go diff --git a/src/core/api/repository.go b/src/core/api/repository.go index 59f87f200..5a71f4151 100644 --- a/src/core/api/repository.go +++ b/src/core/api/repository.go @@ -333,9 +333,14 @@ func (ra *RepositoryAPI) Delete() { Resource: &model.Resource{ Type: model.ResourceTypeRepository, Metadata: &model.ResourceMetadata{ - Name: repoName, - Namespace: projectName, - Vtags: []string{tag}, + Namespace: &model.Namespace{ + Name: projectName, + // TODO filling the metadata + }, + Repository: &model.Repository{ + Name: strings.TrimPrefix(repoName, projectName+"/"), + }, + Vtags: []string{tag}, }, Deleted: true, }, diff --git a/src/core/service/notifications/registry/handler.go b/src/core/service/notifications/registry/handler.go index d68e1b2dc..263766178 100644 --- a/src/core/service/notifications/registry/handler.go +++ b/src/core/service/notifications/registry/handler.go @@ -120,9 +120,14 @@ func (n *NotificationHandler) Post() { Resource: &model.Resource{ Type: model.ResourceTypeRepository, Metadata: &model.ResourceMetadata{ - Name: repository, - Namespace: project, - Vtags: []string{tag}, + Namespace: &model.Namespace{ + Name: project, + // TODO filling the metadata + }, + Repository: &model.Repository{ + Name: strings.TrimPrefix(repository, project+"/"), + }, + Vtags: []string{tag}, }, }, } diff --git a/src/replication/ng/adapter/adapter.go b/src/replication/ng/adapter/adapter.go index 3fb70ffbf..7b18d85ac 100644 --- a/src/replication/ng/adapter/adapter.go +++ b/src/replication/ng/adapter/adapter.go @@ -33,12 +33,15 @@ type Adapter interface { // Lists the available namespaces under the specified registry with the // provided credential/token ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) - // Create a new namespace - // This method should guarantee it's idempotent - // And returns nil if a namespace with the same name already exists - CreateNamespace(*model.Namespace) error + // ConvertResourceMetadata converts the namespace and repository part of the resource metadata + // to the one that the adapter can handle + ConvertResourceMetadata(*model.ResourceMetadata, *model.Namespace) (*model.ResourceMetadata, error) + // PrepareForPush does the prepare work that needed for pushing/uploading the resource + // eg: create the namespace or repository + PrepareForPush(*model.Resource) error // Get the namespace specified by the name, the returning value should // contain the metadata about the namespace if it has + // TODO remove this method? GetNamespace(string) (*model.Namespace, error) // HealthCheck checks health status of registry HealthCheck() (model.HealthStatus, error) diff --git a/src/replication/ng/adapter/harbor/adapter.go b/src/replication/ng/adapter/harbor/adapter.go index 54dfd805b..3a6eb049e 100644 --- a/src/replication/ng/adapter/harbor/adapter.go +++ b/src/replication/ng/adapter/harbor/adapter.go @@ -15,8 +15,10 @@ package harbor import ( + "errors" "fmt" "net/http" + "strings" common_http "github.com/goharbor/harbor/src/common/http" "github.com/goharbor/harbor/src/common/http/modifier" @@ -85,7 +87,8 @@ func newAdapter(registry *model.Registry) *adapter { func (a *adapter) Info() (*model.RegistryInfo, error) { info := &model.RegistryInfo{ - Type: model.RegistryTypeHarbor, + Type: model.RegistryTypeHarbor, + SupportNamespace: true, SupportedResourceTypes: []model.ResourceType{ model.ResourceTypeRepository, }, @@ -126,13 +129,63 @@ func (a *adapter) Info() (*model.RegistryInfo, error) { func (a *adapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) { return nil, nil } -func (a *adapter) CreateNamespace(namespace *model.Namespace) error { +func (a *adapter) ConvertResourceMetadata(metadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) { + if metadata == nil { + return nil, errors.New("the metadata cannot be null") + } + name := metadata.GetResourceName() + strs := strings.SplitN(name, "/", 2) + if len(strs) < 2 { + return nil, fmt.Errorf("unsupported resource name %s, at least contains one '/'", name) + } + meta := &model.ResourceMetadata{ + Vtags: metadata.Vtags, + Labels: metadata.Labels, + } + meta.Namespace = &model.Namespace{ + Name: strs[0], + } + if metadata.Namespace != nil { + meta.Namespace.Metadata = metadata.Namespace.Metadata + } + meta.Repository = &model.Repository{ + Name: strs[1], + } + if metadata.Repository != nil { + meta.Repository.Metadata = metadata.Repository.Metadata + } + // replace the namespace if it is specified + if namespace == nil || len(namespace.Name) == 0 { + return meta, nil + } + if strings.Contains(namespace.Name, "/") { + return nil, fmt.Errorf("the namespace %s cannot contain '/'", namespace.Name) + } + meta.Namespace.Name = namespace.Name + if namespace.Metadata != nil { + meta.Namespace.Metadata = namespace.Metadata + } + return meta, nil +} +func (a *adapter) PrepareForPush(resource *model.Resource) error { + if resource == nil { + return errors.New("the resource cannot be null") + } + if resource.Metadata == nil { + return errors.New("the metadata of resource cannot be null") + } + if resource.Metadata.Namespace == nil { + return errors.New("the namespace of resource cannot be null") + } + if len(resource.Metadata.Namespace.Name) == 0 { + return errors.New("the name of the namespace cannot be null") + } project := &struct { Name string `json:"project_name"` Metadata map[string]interface{} `json:"metadata"` }{ - Name: namespace.Name, - Metadata: namespace.Metadata, + Name: resource.Metadata.Namespace.Name, + Metadata: resource.Metadata.Namespace.Metadata, } // TODO @@ -160,11 +213,13 @@ func (a *adapter) CreateNamespace(namespace *model.Namespace) error { err := a.client.Post(a.coreServiceURL+"/api/projects", project) if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict { - log.Debugf("got 409 when trying to create project %s", namespace.Name) + log.Debugf("got 409 when trying to create project %s", resource.Metadata.Namespace.Name) return nil } return err } + +// TODO remove this method func (a *adapter) GetNamespace(namespace string) (*model.Namespace, error) { project, err := a.getProject(namespace) if err != nil { diff --git a/src/replication/ng/adapter/harbor/adapter_test.go b/src/replication/ng/adapter/harbor/adapter_test.go index 716a7005d..4efa098ca 100644 --- a/src/replication/ng/adapter/harbor/adapter_test.go +++ b/src/replication/ng/adapter/harbor/adapter_test.go @@ -18,12 +18,10 @@ import ( "net/http" "testing" - "github.com/stretchr/testify/assert" - - "github.com/stretchr/testify/require" - "github.com/goharbor/harbor/src/common/utils/test" "github.com/goharbor/harbor/src/replication/ng/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestInfo(t *testing.T) { @@ -77,8 +75,7 @@ func TestListNamespaces(t *testing.T) { // TODO } -func TestCreateNamespace(t *testing.T) { - // project doesn't exist +func TestPrepareForPush(t *testing.T) { server := test.NewServer(&test.RequestHandlerMapping{ Method: http.MethodPost, Pattern: "/api/projects", @@ -90,10 +87,41 @@ func TestCreateNamespace(t *testing.T) { URL: server.URL, } adapter := newAdapter(registry) - err := adapter.CreateNamespace(&model.Namespace{ - Name: "library", + // nil resource + err := adapter.PrepareForPush(nil) + require.NotNil(t, err) + // nil metadata + err = adapter.PrepareForPush(&model.Resource{}) + require.NotNil(t, err) + // nil namespace + err = adapter.PrepareForPush(&model.Resource{ + Metadata: &model.ResourceMetadata{}, + }) + require.NotNil(t, err) + // nil namespace name + err = adapter.PrepareForPush(&model.Resource{ + Metadata: &model.ResourceMetadata{ + Namespace: &model.Namespace{}, + }, + }) + require.NotNil(t, err) + // nil namespace name + err = adapter.PrepareForPush(&model.Resource{ + Metadata: &model.ResourceMetadata{ + Namespace: &model.Namespace{}, + }, + }) + require.NotNil(t, err) + // project doesn't exist + err = adapter.PrepareForPush(&model.Resource{ + Metadata: &model.ResourceMetadata{ + Namespace: &model.Namespace{ + Name: "library", + }, + }, }) require.Nil(t, err) + server.Close() // project already exists @@ -108,11 +136,14 @@ func TestCreateNamespace(t *testing.T) { URL: server.URL, } adapter = newAdapter(registry) - err = adapter.CreateNamespace(&model.Namespace{ - Name: "library", + err = adapter.PrepareForPush(&model.Resource{ + Metadata: &model.ResourceMetadata{ + Namespace: &model.Namespace{ + Name: "library", + }, + }, }) require.Nil(t, err) - server.Close() } func TestGetNamespace(t *testing.T) { diff --git a/src/replication/ng/adapter/harbor/chart_registry.go b/src/replication/ng/adapter/harbor/chart_registry.go index 6e24679a8..27200d4a6 100644 --- a/src/replication/ng/adapter/harbor/chart_registry.go +++ b/src/replication/ng/adapter/harbor/chart_registry.go @@ -67,9 +67,14 @@ func (a *adapter) FetchCharts(namespaces []string, filters []*model.Filter) ([]* Type: model.ResourceTypeChart, Registry: a.registry, Metadata: &model.ResourceMetadata{ - Namespace: namespace, - Name: fmt.Sprintf("%s/%s", namespace, chart.Name), - Vtags: []string{version.Version}, + Namespace: &model.Namespace{ + Name: namespace, + // TODO filling the metadata + }, + Repository: &model.Repository{ + Name: chart.Name, + }, + Vtags: []string{version.Version}, }, }) } diff --git a/src/replication/ng/adapter/harbor/chart_registry_test.go b/src/replication/ng/adapter/harbor/chart_registry_test.go index 2c7637b51..c3a348039 100644 --- a/src/replication/ng/adapter/harbor/chart_registry_test.go +++ b/src/replication/ng/adapter/harbor/chart_registry_test.go @@ -61,8 +61,8 @@ func TestFetchCharts(t *testing.T) { require.Nil(t, err) assert.Equal(t, 2, len(resources)) assert.Equal(t, model.ResourceTypeChart, resources[0].Type) - assert.Equal(t, "library/harbor", resources[0].Metadata.Name) - assert.Equal(t, "library", resources[0].Metadata.Namespace) + assert.Equal(t, "harbor", resources[0].Metadata.Repository.Name) + assert.Equal(t, "library", resources[0].Metadata.Namespace.Name) assert.Equal(t, 1, len(resources[0].Metadata.Vtags)) assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0]) } diff --git a/src/replication/ng/adapter/harbor/image_registry.go b/src/replication/ng/adapter/harbor/image_registry.go index def13b2e0..787222f87 100644 --- a/src/replication/ng/adapter/harbor/image_registry.go +++ b/src/replication/ng/adapter/harbor/image_registry.go @@ -16,6 +16,7 @@ package harbor import ( "fmt" + "strings" "github.com/goharbor/harbor/src/replication/ng/model" ) @@ -56,9 +57,14 @@ func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]* Type: model.ResourceTypeRepository, Registry: a.registry, Metadata: &model.ResourceMetadata{ - Namespace: namespace, - Name: repository.Name, - Vtags: vtags, + Namespace: &model.Namespace{ + Name: namespace, + // TODO filling the metadata + }, + Repository: &model.Repository{ + Name: strings.TrimPrefix(repository.Name, namespace+"/"), + }, + Vtags: vtags, }, }) } diff --git a/src/replication/ng/adapter/harbor/image_registry_test.go b/src/replication/ng/adapter/harbor/image_registry_test.go index 37d8d3680..40e2e2c68 100644 --- a/src/replication/ng/adapter/harbor/image_registry_test.go +++ b/src/replication/ng/adapter/harbor/image_registry_test.go @@ -71,8 +71,8 @@ func TestFetchImages(t *testing.T) { require.Nil(t, err) assert.Equal(t, 1, len(resources)) assert.Equal(t, model.ResourceTypeRepository, resources[0].Type) - assert.Equal(t, "library/hello-world", resources[0].Metadata.Name) - assert.Equal(t, "library", resources[0].Metadata.Namespace) + assert.Equal(t, "hello-world", resources[0].Metadata.Repository.Name) + assert.Equal(t, "library", resources[0].Metadata.Namespace.Name) assert.Equal(t, 2, len(resources[0].Metadata.Vtags)) assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0]) assert.Equal(t, "2.0", resources[0].Metadata.Vtags[1]) diff --git a/src/replication/ng/event/handler.go b/src/replication/ng/event/handler.go index d9d078cb0..7b33af004 100644 --- a/src/replication/ng/event/handler.go +++ b/src/replication/ng/event/handler.go @@ -57,9 +57,9 @@ func (h *handler) Handle(event *Event) error { var err error switch event.Type { case EventTypeImagePush, EventTypeChartUpload: - policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace) + policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace.Name) case EventTypeImageDelete, EventTypeChartDelete: - policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace, true) + policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace.Name, true) default: return fmt.Errorf("unsupported event type %s", event.Type) } diff --git a/src/replication/ng/event/handler_test.go b/src/replication/ng/event/handler_test.go index 59a26e153..c7ab97440 100644 --- a/src/replication/ng/event/handler_test.go +++ b/src/replication/ng/event/handler_test.go @@ -161,9 +161,13 @@ func TestHandle(t *testing.T) { err = handler.Handle(&Event{ Resource: &model.Resource{ Metadata: &model.ResourceMetadata{ - Name: "library/hello-world", - Namespace: "library", - Vtags: []string{}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "hello-world", + }, + Vtags: []string{}, }, }, Type: EventTypeImagePush, @@ -174,9 +178,13 @@ func TestHandle(t *testing.T) { err = handler.Handle(&Event{ Resource: &model.Resource{ Metadata: &model.ResourceMetadata{ - Name: "library/hello-world", - Namespace: "library", - Vtags: []string{"latest"}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "hello-world", + }, + Vtags: []string{"latest"}, }, }, Type: "unsupported", @@ -187,9 +195,13 @@ func TestHandle(t *testing.T) { err = handler.Handle(&Event{ Resource: &model.Resource{ Metadata: &model.ResourceMetadata{ - Name: "library/hello-world", - Namespace: "library", - Vtags: []string{"latest"}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "hello-world", + }, + Vtags: []string{"latest"}, }, }, Type: EventTypeImagePush, @@ -200,9 +212,13 @@ func TestHandle(t *testing.T) { err = handler.Handle(&Event{ Resource: &model.Resource{ Metadata: &model.ResourceMetadata{ - Name: "library/hello-world", - Namespace: "library", - Vtags: []string{"latest"}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "hello-world", + }, + Vtags: []string{"latest"}, }, }, Type: EventTypeImageDelete, diff --git a/src/replication/ng/model/registry.go b/src/replication/ng/model/registry.go index 2e53500e6..2dae48da5 100644 --- a/src/replication/ng/model/registry.go +++ b/src/replication/ng/model/registry.go @@ -108,6 +108,7 @@ type FilterStyle struct { type RegistryInfo struct { Type RegistryType `json:"type"` Description string `json:"description"` + SupportNamespace bool `json:"support_namespace"` SupportedResourceTypes []ResourceType `json:"-"` SupportedResourceFilters []*FilterStyle `json:"supported_resource_filters"` SupportedTriggers []TriggerType `json:"supported_triggers"` diff --git a/src/replication/ng/model/resource.go b/src/replication/ng/model/resource.go index b75d1a964..34dd0c911 100644 --- a/src/replication/ng/model/resource.go +++ b/src/replication/ng/model/resource.go @@ -30,10 +30,32 @@ func (r ResourceType) Valid() bool { // ResourceMetadata of resource type ResourceMetadata struct { - Namespace string `json:"namespace"` - Name string `json:"name"` - Vtags []string `json:"v_tags"` - Labels []string `json:"labels"` + Namespace *Namespace `json:"namespace"` + Repository *Repository `json:"repository"` + Vtags []string `json:"v_tags"` + // TODO the labels should be put into tag and repository level? + Labels []string `json:"labels"` +} + +// GetResourceName returns the name of the resource +func (r *ResourceMetadata) GetResourceName() string { + name := "" + if r.Namespace != nil && len(r.Namespace.Name) > 0 { + name += r.Namespace.Name + } + if r.Repository != nil && len(r.Repository.Name) > 0 { + if len(name) > 0 { + name += "/" + } + name += r.Repository.Name + } + return name +} + +// Repository info of the resource +type Repository struct { + Name string `json:"name"` + Metadata map[string]interface{} `json:"metadata"` } // Resource represents the general replicating content diff --git a/src/replication/ng/model/resource_test.go b/src/replication/ng/model/resource_test.go new file mode 100644 index 000000000..c1c2e7af3 --- /dev/null +++ b/src/replication/ng/model/resource_test.go @@ -0,0 +1,50 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetResourceName(t *testing.T) { + r := &ResourceMetadata{} + assert.Equal(t, "", r.GetResourceName()) + + r = &ResourceMetadata{ + Namespace: &Namespace{ + Name: "library", + }, + } + assert.Equal(t, "library", r.GetResourceName()) + + r = &ResourceMetadata{ + Repository: &Repository{ + Name: "hello-world", + }, + } + assert.Equal(t, "hello-world", r.GetResourceName()) + + r = &ResourceMetadata{ + Namespace: &Namespace{ + Name: "library", + }, + Repository: &Repository{ + Name: "hello-world", + }, + } + assert.Equal(t, "library/hello-world", r.GetResourceName()) +} diff --git a/src/replication/ng/operation/controller.go b/src/replication/ng/operation/controller.go index 6ac2a0b57..ebcd76eda 100644 --- a/src/replication/ng/operation/controller.go +++ b/src/replication/ng/operation/controller.go @@ -104,8 +104,9 @@ func (c *controller) createFlow(executionID int64, policy *model.Policy, resourc Value: resource.Type, }, { - Type: model.FilterTypeName, - Value: resource.Metadata.Name, + Type: model.FilterTypeName, + // TODO only filter the repo part? + Value: resource.Metadata.GetResourceName(), }, { Type: model.FilterTypeTag, diff --git a/src/replication/ng/operation/controller_test.go b/src/replication/ng/operation/controller_test.go index dbe599352..75edb8060 100644 --- a/src/replication/ng/operation/controller_test.go +++ b/src/replication/ng/operation/controller_test.go @@ -132,7 +132,18 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) { func (f *fakedAdapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) { return nil, nil } -func (f *fakedAdapter) CreateNamespace(*model.Namespace) error { +func (f *fakedAdapter) ConvertResourceMetadata(*model.ResourceMetadata, *model.Namespace) (*model.ResourceMetadata, error) { + return &model.ResourceMetadata{ + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "hello-world", + }, + Vtags: []string{"latest"}, + }, nil +} +func (f *fakedAdapter) PrepareForPush(*model.Resource) error { return nil } func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) { @@ -155,9 +166,13 @@ func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter) { Type: model.ResourceTypeRepository, Metadata: &model.ResourceMetadata{ - Name: "library/hello-world", - Namespace: "library", - Vtags: []string{"latest"}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "hello-world", + }, + Vtags: []string{"latest"}, }, Override: false, }, @@ -190,9 +205,13 @@ func (f *fakedAdapter) FetchCharts(namespaces []string, filters []*model.Filter) { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Name: "library/harbor", - Namespace: "library", - Vtags: []string{"0.2.0"}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "harbor", + }, + Vtags: []string{"0.2.0"}, }, }, }, nil @@ -232,7 +251,12 @@ func TestStartReplication(t *testing.T) { resource := &model.Resource{ Type: model.ResourceTypeRepository, Metadata: &model.ResourceMetadata{ - Name: "library/hello-world", + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "hello-world", + }, Vtags: []string{"1.0", "2.0"}, }, } diff --git a/src/replication/ng/operation/flow/copy.go b/src/replication/ng/operation/flow/copy.go index f5e29ffc9..787e2a49d 100644 --- a/src/replication/ng/operation/flow/copy.go +++ b/src/replication/ng/operation/flow/copy.go @@ -57,14 +57,13 @@ func (c *copyFlow) Run(interface{}) (int, error) { log.Infof("no resources need to be replicated for the execution %d, skip", c.executionID) return 0, nil } - dstNamespaces, err := assembleDestinationNamespaces(srcAdapter, srcResources, c.policy.DestNamespace) + dstResources, err := assembleDestinationResources(dstAdapter, srcResources, c.policy) if err != nil { return 0, err } - if err = createNamespaces(dstAdapter, dstNamespaces); err != nil { + if err = prepareForPush(dstAdapter, dstResources); err != nil { return 0, err } - dstResources := assembleDestinationResources(srcResources, c.policy.DestRegistry, c.policy.DestNamespace, c.policy.Override) items, err := preprocess(c.scheduler, srcResources, dstResources) if err != nil { return 0, err diff --git a/src/replication/ng/operation/flow/deletion.go b/src/replication/ng/operation/flow/deletion.go index 68c1af502..706a6f617 100644 --- a/src/replication/ng/operation/flow/deletion.go +++ b/src/replication/ng/operation/flow/deletion.go @@ -43,6 +43,10 @@ func NewDeletionFlow(executionMgr execution.Manager, scheduler scheduler.Schedul } func (d *deletionFlow) Run(interface{}) (int, error) { + _, dstAdapter, err := initialize(d.policy) + if err != nil { + return 0, err + } // filling the registry information for _, resource := range d.resources { resource.Registry = d.policy.SrcRegistry @@ -56,7 +60,10 @@ func (d *deletionFlow) Run(interface{}) (int, error) { log.Infof("no resources need to be replicated for the execution %d, skip", d.executionID) return 0, nil } - dstResources := assembleDestinationResources(srcResources, d.policy.DestRegistry, d.policy.DestNamespace, d.policy.Override) + dstResources, err := assembleDestinationResources(dstAdapter, srcResources, d.policy) + if err != nil { + return 0, err + } items, err := preprocess(d.scheduler, srcResources, dstResources) if err != nil { return 0, err diff --git a/src/replication/ng/operation/flow/deletion_test.go b/src/replication/ng/operation/flow/deletion_test.go index 6644efe63..303328b47 100644 --- a/src/replication/ng/operation/flow/deletion_test.go +++ b/src/replication/ng/operation/flow/deletion_test.go @@ -37,9 +37,13 @@ func TestRunOfDeletionFlow(t *testing.T) { resources := []*model.Resource{ { Metadata: &model.ResourceMetadata{ - Name: "library/hello-world", - Namespace: "library", - Vtags: []string{"latest"}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "hello-world", + }, + Vtags: []string{"latest"}, }, }, } diff --git a/src/replication/ng/operation/flow/stage.go b/src/replication/ng/operation/flow/stage.go index ae34b94de..a354de5e7 100644 --- a/src/replication/ng/operation/flow/stage.go +++ b/src/replication/ng/operation/flow/stage.go @@ -134,7 +134,8 @@ func filterResources(resources []*model.Resource, filters []*model.Filter) ([]*m match = false break } - m, err := util.Match(pattern, resource.Metadata.Name) + // TODO filter only the repository part? + m, err := util.Match(pattern, resource.Metadata.GetResourceName()) if err != nil { return nil, err } @@ -181,73 +182,46 @@ func filterResources(resources []*model.Resource, filters []*model.Filter) ([]*m return res, nil } -// Assemble the namespaces that need to be created on the destination registry: -// step 1: get the detail information for each of the source namespaces -// step 2: if the destination namespace isn't specified in the policy, then the -// same namespaces with the source will be returned. If it is specified, then -// returns the specified one with the merged metadatas of all source namespaces -func assembleDestinationNamespaces(srcAdapter adp.Adapter, srcResources []*model.Resource, dstNamespace string) ([]*model.Namespace, error) { - namespaces := []*model.Namespace{} - for _, srcResource := range srcResources { - namespace, err := srcAdapter.GetNamespace(srcResource.Metadata.Namespace) - if err != nil { - return nil, err - } - namespaces = append(namespaces, namespace) - } - - if len(dstNamespace) != 0 { - namespaces = []*model.Namespace{ - { - Name: dstNamespace, - // TODO merge the metadata - Metadata: map[string]interface{}{}, - }, - } - } - - log.Debug("assemble the destination namespaces completed") - return namespaces, nil -} - -// create the namespaces on the destination registry -func createNamespaces(adapter adp.Adapter, namespaces []*model.Namespace) error { - for _, namespace := range namespaces { - if err := adapter.CreateNamespace(namespace); err != nil { - return fmt.Errorf("failed to create the namespace %s on the destination registry: %v", namespace.Name, err) - } - log.Debugf("namespace %s created on the destination registry", namespace.Name) - } - return nil -} - -// assemble the destination resources by filling the registry, namespace and override properties -func assembleDestinationResources(resources []*model.Resource, - registry *model.Registry, namespace string, override bool) []*model.Resource { +// assemble the destination resources by filling the metadata, registry and override properties +func assembleDestinationResources(adapter adp.Adapter, resources []*model.Resource, + policy *model.Policy) ([]*model.Resource, error) { result := []*model.Resource{} + var namespace *model.Namespace + if len(policy.DestNamespace) > 0 { + namespace = &model.Namespace{ + Name: policy.DestNamespace, + } + } for _, resource := range resources { + metadata, err := adapter.ConvertResourceMetadata(resource.Metadata, namespace) + if err != nil { + return nil, fmt.Errorf("failed to convert the resource metadata of %s: %v", resource.Metadata.GetResourceName(), err) + } res := &model.Resource{ - Type: resource.Type, - Metadata: &model.ResourceMetadata{ - Name: resource.Metadata.Name, - Namespace: resource.Metadata.Namespace, - Vtags: resource.Metadata.Vtags, - }, - Registry: registry, + Type: resource.Type, + Metadata: metadata, + Registry: policy.DestRegistry, ExtendedInfo: resource.ExtendedInfo, Deleted: resource.Deleted, - Override: override, - } - // if the destination namespace is specified, use the specified one - if len(namespace) > 0 { - res.Metadata.Name = strings.Replace(resource.Metadata.Name, - resource.Metadata.Namespace, namespace, 1) - res.Metadata.Namespace = namespace + Override: policy.Override, } result = append(result, res) } log.Debug("assemble the destination resources completed") - return result + return result, nil +} + +// do the prepare work for pushing/uploading the resources: create the namespace or repository +func prepareForPush(adapter adp.Adapter, resources []*model.Resource) error { + // TODO need to consider how to handle that both contains public/private namespace + for _, resource := range resources { + name := resource.Metadata.GetResourceName() + if err := adapter.PrepareForPush(resource); err != nil { + return fmt.Errorf("failed to do the prepare work for pushing/uploading %s: %v", name, err) + } + log.Debugf("the prepare work for pushing/uploading %s completed", name) + } + return nil } // preprocess @@ -341,7 +315,7 @@ func getResourceName(res *model.Resource) string { return "" } if len(meta.Vtags) == 0 { - return meta.Name + return meta.GetResourceName() } - return meta.Name + ":[" + strings.Join(meta.Vtags, ",") + "]" + return meta.GetResourceName() + ":[" + strings.Join(meta.Vtags, ",") + "]" } diff --git a/src/replication/ng/operation/flow/stage_test.go b/src/replication/ng/operation/flow/stage_test.go index 0d31e6d40..a6e3dfb32 100644 --- a/src/replication/ng/operation/flow/stage_test.go +++ b/src/replication/ng/operation/flow/stage_test.go @@ -49,7 +49,14 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) { func (f *fakedAdapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) { return nil, nil } -func (f *fakedAdapter) CreateNamespace(*model.Namespace) error { +func (f *fakedAdapter) ConvertResourceMetadata(metadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) { + if namespace != nil { + metadata.Namespace = namespace + } + return metadata, nil +} + +func (f *fakedAdapter) PrepareForPush(*model.Resource) error { return nil } func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) { @@ -72,9 +79,13 @@ func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter) { Type: model.ResourceTypeRepository, Metadata: &model.ResourceMetadata{ - Name: "library/hello-world", - Namespace: "library", - Vtags: []string{"latest"}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "hello-world", + }, + Vtags: []string{"latest"}, }, Override: false, }, @@ -107,9 +118,13 @@ func (f *fakedAdapter) FetchCharts(namespaces []string, filters []*model.Filter) { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Name: "library/harbor", - Namespace: "library", - Vtags: []string{"0.2.0"}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "harbor", + }, + Vtags: []string{"0.2.0"}, }, }, }, nil @@ -226,9 +241,13 @@ func TestFilterResources(t *testing.T) { { Type: model.ResourceTypeRepository, Metadata: &model.ResourceMetadata{ - Name: "library/hello-world", - Namespace: "library", - Vtags: []string{"latest"}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "hello-world", + }, + Vtags: []string{"latest"}, // TODO test labels Labels: nil, }, @@ -238,9 +257,13 @@ func TestFilterResources(t *testing.T) { { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Name: "library/harbor", - Namespace: "library", - Vtags: []string{"0.2.0", "0.3.0"}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "harbor", + }, + Vtags: []string{"0.2.0", "0.3.0"}, // TODO test labels Labels: nil, }, @@ -250,9 +273,13 @@ func TestFilterResources(t *testing.T) { { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Name: "library/mysql", - Namespace: "library", - Vtags: []string{"1.0"}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "mysql", + }, + Vtags: []string{"1.0"}, // TODO test labels Labels: nil, }, @@ -281,67 +308,40 @@ func TestFilterResources(t *testing.T) { res, err := filterResources(resources, filters) require.Nil(t, err) assert.Equal(t, 1, len(res)) - assert.Equal(t, "library/harbor", res[0].Metadata.Name) + assert.Equal(t, "library", res[0].Metadata.Namespace.Name) + assert.Equal(t, "harbor", res[0].Metadata.Repository.Name) assert.Equal(t, 1, len(res[0].Metadata.Vtags)) assert.Equal(t, "0.2.0", res[0].Metadata.Vtags[0]) } -func TestAssembleDestinationNamespaces(t *testing.T) { - adapter := &fakedAdapter{} - resources := []*model.Resource{ - { - Metadata: &model.ResourceMetadata{ - Namespace: "library", - }, - }, - } - namespace := "" - ns, err := assembleDestinationNamespaces(adapter, resources, namespace) - require.Nil(t, err) - assert.Equal(t, 1, len(ns)) - assert.Equal(t, "library", ns[0].Name) - assert.Equal(t, true, ns[0].Metadata["public"].(bool)) - - namespace = "test" - ns, err = assembleDestinationNamespaces(adapter, resources, namespace) - require.Nil(t, err) - assert.Equal(t, 1, len(ns)) - assert.Equal(t, "test", ns[0].Name) - // TODO add test for merged metadata - // assert.Equal(t, true, ns[0].Metadata["public"].(bool)) -} - -func TestCreateNamespaces(t *testing.T) { - adapter := &fakedAdapter{} - namespaces := []*model.Namespace{ - { - Name: "library", - }, - } - err := createNamespaces(adapter, namespaces) - require.Nil(t, err) -} - func TestAssembleDestinationResources(t *testing.T) { + adapter := &fakedAdapter{} resources := []*model.Resource{ { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Name: "library/hello-world", - Namespace: "library", - Vtags: []string{"latest"}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "hello-world", + }, + Vtags: []string{"latest"}, }, Override: false, }, } - registry := &model.Registry{} - namespace := "test" - override := true - res := assembleDestinationResources(resources, registry, namespace, override) + policy := &model.Policy{ + DestRegistry: &model.Registry{}, + DestNamespace: "test", + Override: true, + } + res, err := assembleDestinationResources(adapter, resources, policy) + require.Nil(t, err) assert.Equal(t, 1, len(res)) assert.Equal(t, model.ResourceTypeChart, res[0].Type) - assert.Equal(t, "test/hello-world", res[0].Metadata.Name) - assert.Equal(t, namespace, res[0].Metadata.Namespace) + assert.Equal(t, "hello-world", res[0].Metadata.Repository.Name) + assert.Equal(t, "test", res[0].Metadata.Namespace.Name) assert.Equal(t, 1, len(res[0].Metadata.Vtags)) assert.Equal(t, "latest", res[0].Metadata.Vtags[0]) } @@ -352,9 +352,13 @@ func TestPreprocess(t *testing.T) { { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Name: "library/hello-world", - Namespace: "library", - Vtags: []string{"latest"}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "hello-world", + }, + Vtags: []string{"latest"}, }, Override: false, }, @@ -363,9 +367,13 @@ func TestPreprocess(t *testing.T) { { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Name: "test/hello-world", - Namespace: "test", - Vtags: []string{"latest"}, + Namespace: &model.Namespace{ + Name: "test", + }, + Repository: &model.Repository{ + Name: "hello-world", + }, + Vtags: []string{"latest"}, }, Override: false, }, diff --git a/src/replication/ng/operation/scheduler/scheduler_test.go b/src/replication/ng/operation/scheduler/scheduler_test.go index 1ff75b6e8..f0878598d 100644 --- a/src/replication/ng/operation/scheduler/scheduler_test.go +++ b/src/replication/ng/operation/scheduler/scheduler_test.go @@ -50,9 +50,11 @@ func TestStop(t *testing.T) { func generateData() ([]*ScheduleItem, error) { srcResource := &model.Resource{ Metadata: &model.ResourceMetadata{ - Namespace: "namespace1", - Vtags: []string{"latest"}, - Labels: []string{"latest"}, + Namespace: &model.Namespace{ + Name: "namespace1", + }, + Vtags: []string{"latest"}, + Labels: []string{"latest"}, }, Registry: &model.Registry{ Credential: &model.Credential{}, @@ -60,9 +62,11 @@ func generateData() ([]*ScheduleItem, error) { } destResource := &model.Resource{ Metadata: &model.ResourceMetadata{ - Namespace: "namespace2", - Vtags: []string{"v1", "v2"}, - Labels: []string{"latest"}, + Namespace: &model.Namespace{ + Name: "namespace2", + }, + Vtags: []string{"v1", "v2"}, + Labels: []string{"latest"}, }, Registry: &model.Registry{ Credential: &model.Credential{}, diff --git a/src/replication/ng/transfer/chart/transfer.go b/src/replication/ng/transfer/chart/transfer.go index 90558b7b9..38144ee72 100644 --- a/src/replication/ng/transfer/chart/transfer.go +++ b/src/replication/ng/transfer/chart/transfer.go @@ -17,10 +17,9 @@ package chart import ( "errors" - "github.com/goharbor/harbor/src/replication/ng/adapter" - "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/jobservice/errs" + "github.com/goharbor/harbor/src/replication/ng/adapter" "github.com/goharbor/harbor/src/replication/ng/model" trans "github.com/goharbor/harbor/src/replication/ng/transfer" ) @@ -63,17 +62,17 @@ func (t *transfer) Transfer(src *model.Resource, dst *model.Resource) error { // delete the chart on destination registry if dst.Deleted { return t.delete(&chart{ - name: dst.Metadata.Name, + name: dst.Metadata.GetResourceName(), version: dst.Metadata.Vtags[0], }) } srcChart := &chart{ - name: src.Metadata.Name, + name: src.Metadata.GetResourceName(), version: src.Metadata.Vtags[0], } dstChart := &chart{ - name: dst.Metadata.Name, + name: dst.Metadata.GetResourceName(), version: dst.Metadata.Vtags[0], } // copy the chart from source registry to the destination diff --git a/src/replication/ng/transfer/chart/transfer_test.go b/src/replication/ng/transfer/chart/transfer_test.go index 961dc01a8..1edbebf7e 100644 --- a/src/replication/ng/transfer/chart/transfer_test.go +++ b/src/replication/ng/transfer/chart/transfer_test.go @@ -34,9 +34,13 @@ func (f *fakeRegistry) FetchCharts(namespaces []string, filters []*model.Filter) { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Name: "library/harbor", - Namespace: "library", - Vtags: []string{"0.2.0"}, + Namespace: &model.Namespace{ + Name: "library", + }, + Repository: &model.Repository{ + Name: "harbor", + }, + Vtags: []string{"0.2.0"}, }, }, }, nil diff --git a/src/replication/ng/transfer/repository/transfer.go b/src/replication/ng/transfer/repository/transfer.go index 8719e0d8b..fa29c3002 100644 --- a/src/replication/ng/transfer/repository/transfer.go +++ b/src/replication/ng/transfer/repository/transfer.go @@ -18,13 +18,12 @@ import ( "errors" "strings" - "github.com/goharbor/harbor/src/replication/ng/adapter" - "github.com/docker/distribution" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/jobservice/errs" + "github.com/goharbor/harbor/src/replication/ng/adapter" "github.com/goharbor/harbor/src/replication/ng/model" trans "github.com/goharbor/harbor/src/replication/ng/transfer" ) @@ -67,17 +66,17 @@ func (t *transfer) Transfer(src *model.Resource, dst *model.Resource) error { // delete the repository on destination registry if dst.Deleted { return t.delete(&repository{ - repository: dst.Metadata.Name, + repository: dst.Metadata.GetResourceName(), tags: dst.Metadata.Vtags, }) } srcRepo := &repository{ - repository: src.Metadata.Name, + repository: src.Metadata.GetResourceName(), tags: src.Metadata.Vtags, } dstRepo := &repository{ - repository: dst.Metadata.Name, + repository: dst.Metadata.GetResourceName(), tags: dst.Metadata.Vtags, } // copy the repository from source registry to the destination