Update the adapter interface

Add ConvertResourceMetadata and PrepareForPush methods

Signed-off-by: Wenkai Yin <yinw@vmware.com>

src/core/api/repository.go

@@ -333,9 +333,14 @@ func (ra *RepositoryAPI) Delete() {
 				Resource: &model.Resource{
 					Type: model.ResourceTypeRepository,
 					Metadata: &model.ResourceMetadata{
-						Name:      repoName,
+						Namespace: &model.Namespace{
-						Namespace: projectName,
+							Name: projectName,
-						Vtags:     []string{tag},
+							// TODO filling the metadata
+						},
+						Repository: &model.Repository{
+							Name: strings.TrimPrefix(repoName, projectName+"/"),
+						},
+						Vtags: []string{tag},
 					},
 					Deleted: true,
 				},

src/core/service/notifications/registry/handler.go

@@ -120,9 +120,14 @@ func (n *NotificationHandler) Post() {
 					Resource: &model.Resource{
 						Type: model.ResourceTypeRepository,
 						Metadata: &model.ResourceMetadata{
-							Name:      repository,
+							Namespace: &model.Namespace{
-							Namespace: project,
+								Name: project,
-							Vtags:     []string{tag},
+								// TODO filling the metadata
+							},
+							Repository: &model.Repository{
+								Name: strings.TrimPrefix(repository, project+"/"),
+							},
+							Vtags: []string{tag},
 						},
 					},
 				}

src/replication/ng/adapter/adapter.go

@@ -33,12 +33,15 @@ type Adapter interface {
 	// Lists the available namespaces under the specified registry with the
 	// provided credential/token
 	ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error)
-	// Create a new namespace
+	// ConvertResourceMetadata converts the namespace and repository part of the resource metadata
-	// This method should guarantee it's idempotent
+	// to the one that the adapter can handle
-	// And returns nil if a namespace with the same name already exists
+	ConvertResourceMetadata(*model.ResourceMetadata, *model.Namespace) (*model.ResourceMetadata, error)
-	CreateNamespace(*model.Namespace) error
+	// PrepareForPush does the prepare work that needed for pushing/uploading the resource
+	// eg: create the namespace or repository
+	PrepareForPush(*model.Resource) error
 	// Get the namespace specified by the name, the returning value should
 	// contain the metadata about the namespace if it has
+	// TODO remove this method?
 	GetNamespace(string) (*model.Namespace, error)
 	// HealthCheck checks health status of registry
 	HealthCheck() (model.HealthStatus, error)

src/replication/ng/adapter/harbor/adapter.go

@@ -15,8 +15,10 @@
 package harbor
 
 import (
+	"errors"
 	"fmt"
 	"net/http"
+	"strings"
 
 	common_http "github.com/goharbor/harbor/src/common/http"
 	"github.com/goharbor/harbor/src/common/http/modifier"
@@ -85,7 +87,8 @@ func newAdapter(registry *model.Registry) *adapter {
 
 func (a *adapter) Info() (*model.RegistryInfo, error) {
 	info := &model.RegistryInfo{
-		Type: model.RegistryTypeHarbor,
+		Type:             model.RegistryTypeHarbor,
+		SupportNamespace: true,
 		SupportedResourceTypes: []model.ResourceType{
 			model.ResourceTypeRepository,
 		},
@@ -126,13 +129,63 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
 func (a *adapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) {
 	return nil, nil
 }
-func (a *adapter) CreateNamespace(namespace *model.Namespace) error {
+func (a *adapter) ConvertResourceMetadata(metadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) {
+	if metadata == nil {
+		return nil, errors.New("the metadata cannot be null")
+	}
+	name := metadata.GetResourceName()
+	strs := strings.SplitN(name, "/", 2)
+	if len(strs) < 2 {
+		return nil, fmt.Errorf("unsupported resource name %s, at least contains one '/'", name)
+	}
+	meta := &model.ResourceMetadata{
+		Vtags:  metadata.Vtags,
+		Labels: metadata.Labels,
+	}
+	meta.Namespace = &model.Namespace{
+		Name: strs[0],
+	}
+	if metadata.Namespace != nil {
+		meta.Namespace.Metadata = metadata.Namespace.Metadata
+	}
+	meta.Repository = &model.Repository{
+		Name: strs[1],
+	}
+	if metadata.Repository != nil {
+		meta.Repository.Metadata = metadata.Repository.Metadata
+	}
+	// replace the namespace if it is specified
+	if namespace == nil || len(namespace.Name) == 0 {
+		return meta, nil
+	}
+	if strings.Contains(namespace.Name, "/") {
+		return nil, fmt.Errorf("the namespace %s cannot contain '/'", namespace.Name)
+	}
+	meta.Namespace.Name = namespace.Name
+	if namespace.Metadata != nil {
+		meta.Namespace.Metadata = namespace.Metadata
+	}
+	return meta, nil
+}
+func (a *adapter) PrepareForPush(resource *model.Resource) error {
+	if resource == nil {
+		return errors.New("the resource cannot be null")
+	}
+	if resource.Metadata == nil {
+		return errors.New("the metadata of resource cannot be null")
+	}
+	if resource.Metadata.Namespace == nil {
+		return errors.New("the namespace of resource cannot be null")
+	}
+	if len(resource.Metadata.Namespace.Name) == 0 {
+		return errors.New("the name of the namespace cannot be null")
+	}
 	project := &struct {
 		Name     string                 `json:"project_name"`
 		Metadata map[string]interface{} `json:"metadata"`
 	}{
-		Name:     namespace.Name,
+		Name:     resource.Metadata.Namespace.Name,
-		Metadata: namespace.Metadata,
+		Metadata: resource.Metadata.Namespace.Metadata,
 	}
 
 	// TODO
@@ -160,11 +213,13 @@ func (a *adapter) CreateNamespace(namespace *model.Namespace) error {
 
 	err := a.client.Post(a.coreServiceURL+"/api/projects", project)
 	if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
-		log.Debugf("got 409 when trying to create project %s", namespace.Name)
+		log.Debugf("got 409 when trying to create project %s", resource.Metadata.Namespace.Name)
 		return nil
 	}
 	return err
 }
+
+// TODO remove this method
 func (a *adapter) GetNamespace(namespace string) (*model.Namespace, error) {
 	project, err := a.getProject(namespace)
 	if err != nil {

src/replication/ng/adapter/harbor/adapter_test.go

@@ -18,12 +18,10 @@ import (
 	"net/http"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
-
-	"github.com/stretchr/testify/require"
-
 	"github.com/goharbor/harbor/src/common/utils/test"
 	"github.com/goharbor/harbor/src/replication/ng/model"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestInfo(t *testing.T) {
@@ -77,8 +75,7 @@ func TestListNamespaces(t *testing.T) {
 	// TODO
 }
 
-func TestCreateNamespace(t *testing.T) {
+func TestPrepareForPush(t *testing.T) {
-	// project doesn't exist
 	server := test.NewServer(&test.RequestHandlerMapping{
 		Method:  http.MethodPost,
 		Pattern: "/api/projects",
@@ -90,10 +87,41 @@ func TestCreateNamespace(t *testing.T) {
 		URL: server.URL,
 	}
 	adapter := newAdapter(registry)
-	err := adapter.CreateNamespace(&model.Namespace{
+	// nil resource
-		Name: "library",
+	err := adapter.PrepareForPush(nil)
+	require.NotNil(t, err)
+	// nil metadata
+	err = adapter.PrepareForPush(&model.Resource{})
+	require.NotNil(t, err)
+	// nil namespace
+	err = adapter.PrepareForPush(&model.Resource{
+		Metadata: &model.ResourceMetadata{},
+	})
+	require.NotNil(t, err)
+	// nil namespace name
+	err = adapter.PrepareForPush(&model.Resource{
+		Metadata: &model.ResourceMetadata{
+			Namespace: &model.Namespace{},
+		},
+	})
+	require.NotNil(t, err)
+	// nil namespace name
+	err = adapter.PrepareForPush(&model.Resource{
+		Metadata: &model.ResourceMetadata{
+			Namespace: &model.Namespace{},
+		},
+	})
+	require.NotNil(t, err)
+	// project doesn't exist
+	err = adapter.PrepareForPush(&model.Resource{
+		Metadata: &model.ResourceMetadata{
+			Namespace: &model.Namespace{
+				Name: "library",
+			},
+		},
 	})
 	require.Nil(t, err)
+
 	server.Close()
 
 	// project already exists
@@ -108,11 +136,14 @@ func TestCreateNamespace(t *testing.T) {
 		URL: server.URL,
 	}
 	adapter = newAdapter(registry)
-	err = adapter.CreateNamespace(&model.Namespace{
+	err = adapter.PrepareForPush(&model.Resource{
-		Name: "library",
+		Metadata: &model.ResourceMetadata{
+			Namespace: &model.Namespace{
+				Name: "library",
+			},
+		},
 	})
 	require.Nil(t, err)
-	server.Close()
 }
 
 func TestGetNamespace(t *testing.T) {

src/replication/ng/adapter/harbor/chart_registry.go

@@ -67,9 +67,14 @@ func (a *adapter) FetchCharts(namespaces []string, filters []*model.Filter) ([]*
 					Type:     model.ResourceTypeChart,
 					Registry: a.registry,
 					Metadata: &model.ResourceMetadata{
-						Namespace: namespace,
+						Namespace: &model.Namespace{
-						Name:      fmt.Sprintf("%s/%s", namespace, chart.Name),
+							Name: namespace,
-						Vtags:     []string{version.Version},
+							// TODO filling the metadata
+						},
+						Repository: &model.Repository{
+							Name: chart.Name,
+						},
+						Vtags: []string{version.Version},
 					},
 				})
 			}

src/replication/ng/adapter/harbor/chart_registry_test.go

@@ -61,8 +61,8 @@ func TestFetchCharts(t *testing.T) {
 	require.Nil(t, err)
 	assert.Equal(t, 2, len(resources))
 	assert.Equal(t, model.ResourceTypeChart, resources[0].Type)
-	assert.Equal(t, "library/harbor", resources[0].Metadata.Name)
+	assert.Equal(t, "harbor", resources[0].Metadata.Repository.Name)
-	assert.Equal(t, "library", resources[0].Metadata.Namespace)
+	assert.Equal(t, "library", resources[0].Metadata.Namespace.Name)
 	assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
 	assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0])
 }

src/replication/ng/adapter/harbor/image_registry.go

@@ -16,6 +16,7 @@ package harbor
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/goharbor/harbor/src/replication/ng/model"
 )
@@ -56,9 +57,14 @@ func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]*
 				Type:     model.ResourceTypeRepository,
 				Registry: a.registry,
 				Metadata: &model.ResourceMetadata{
-					Namespace: namespace,
+					Namespace: &model.Namespace{
-					Name:      repository.Name,
+						Name: namespace,
-					Vtags:     vtags,
+						// TODO filling the metadata
+					},
+					Repository: &model.Repository{
+						Name: strings.TrimPrefix(repository.Name, namespace+"/"),
+					},
+					Vtags: vtags,
 				},
 			})
 		}

src/replication/ng/adapter/harbor/image_registry_test.go

@@ -71,8 +71,8 @@ func TestFetchImages(t *testing.T) {
 	require.Nil(t, err)
 	assert.Equal(t, 1, len(resources))
 	assert.Equal(t, model.ResourceTypeRepository, resources[0].Type)
-	assert.Equal(t, "library/hello-world", resources[0].Metadata.Name)
+	assert.Equal(t, "hello-world", resources[0].Metadata.Repository.Name)
-	assert.Equal(t, "library", resources[0].Metadata.Namespace)
+	assert.Equal(t, "library", resources[0].Metadata.Namespace.Name)
 	assert.Equal(t, 2, len(resources[0].Metadata.Vtags))
 	assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0])
 	assert.Equal(t, "2.0", resources[0].Metadata.Vtags[1])

src/replication/ng/event/handler.go

@@ -57,9 +57,9 @@ func (h *handler) Handle(event *Event) error {
 	var err error
 	switch event.Type {
 	case EventTypeImagePush, EventTypeChartUpload:
-		policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace)
+		policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace.Name)
 	case EventTypeImageDelete, EventTypeChartDelete:
-		policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace, true)
+		policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace.Name, true)
 	default:
 		return fmt.Errorf("unsupported event type %s", event.Type)
 	}

src/replication/ng/event/handler_test.go

@@ -161,9 +161,13 @@ func TestHandle(t *testing.T) {
 	err = handler.Handle(&Event{
 		Resource: &model.Resource{
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/hello-world",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{},
+				},
+				Repository: &model.Repository{
+					Name: "hello-world",
+				},
+				Vtags: []string{},
 			},
 		},
 		Type: EventTypeImagePush,
@@ -174,9 +178,13 @@ func TestHandle(t *testing.T) {
 	err = handler.Handle(&Event{
 		Resource: &model.Resource{
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/hello-world",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{"latest"},
+				},
+				Repository: &model.Repository{
+					Name: "hello-world",
+				},
+				Vtags: []string{"latest"},
 			},
 		},
 		Type: "unsupported",
@@ -187,9 +195,13 @@ func TestHandle(t *testing.T) {
 	err = handler.Handle(&Event{
 		Resource: &model.Resource{
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/hello-world",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{"latest"},
+				},
+				Repository: &model.Repository{
+					Name: "hello-world",
+				},
+				Vtags: []string{"latest"},
 			},
 		},
 		Type: EventTypeImagePush,
@@ -200,9 +212,13 @@ func TestHandle(t *testing.T) {
 	err = handler.Handle(&Event{
 		Resource: &model.Resource{
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/hello-world",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{"latest"},
+				},
+				Repository: &model.Repository{
+					Name: "hello-world",
+				},
+				Vtags: []string{"latest"},
 			},
 		},
 		Type: EventTypeImageDelete,

src/replication/ng/model/registry.go

@@ -108,6 +108,7 @@ type FilterStyle struct {
 type RegistryInfo struct {
 	Type                     RegistryType   `json:"type"`
 	Description              string         `json:"description"`
+	SupportNamespace         bool           `json:"support_namespace"`
 	SupportedResourceTypes   []ResourceType `json:"-"`
 	SupportedResourceFilters []*FilterStyle `json:"supported_resource_filters"`
 	SupportedTriggers        []TriggerType  `json:"supported_triggers"`

src/replication/ng/model/resource.go

@@ -30,10 +30,32 @@ func (r ResourceType) Valid() bool {
 
 // ResourceMetadata of resource
 type ResourceMetadata struct {
-	Namespace string   `json:"namespace"`
+	Namespace  *Namespace  `json:"namespace"`
-	Name      string   `json:"name"`
+	Repository *Repository `json:"repository"`
-	Vtags     []string `json:"v_tags"`
+	Vtags      []string    `json:"v_tags"`
-	Labels    []string `json:"labels"`
+	// TODO the labels should be put into tag and repository level?
+	Labels []string `json:"labels"`
+}
+
+// GetResourceName returns the name of the resource
+func (r *ResourceMetadata) GetResourceName() string {
+	name := ""
+	if r.Namespace != nil && len(r.Namespace.Name) > 0 {
+		name += r.Namespace.Name
+	}
+	if r.Repository != nil && len(r.Repository.Name) > 0 {
+		if len(name) > 0 {
+			name += "/"
+		}
+		name += r.Repository.Name
+	}
+	return name
+}
+
+// Repository info of the resource
+type Repository struct {
+	Name     string                 `json:"name"`
+	Metadata map[string]interface{} `json:"metadata"`
 }
 
 // Resource represents the general replicating content

src/replication/ng/model/resource_test.go

@@ -0,0 +1,50 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package model
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetResourceName(t *testing.T) {
+	r := &ResourceMetadata{}
+	assert.Equal(t, "", r.GetResourceName())
+
+	r = &ResourceMetadata{
+		Namespace: &Namespace{
+			Name: "library",
+		},
+	}
+	assert.Equal(t, "library", r.GetResourceName())
+
+	r = &ResourceMetadata{
+		Repository: &Repository{
+			Name: "hello-world",
+		},
+	}
+	assert.Equal(t, "hello-world", r.GetResourceName())
+
+	r = &ResourceMetadata{
+		Namespace: &Namespace{
+			Name: "library",
+		},
+		Repository: &Repository{
+			Name: "hello-world",
+		},
+	}
+	assert.Equal(t, "library/hello-world", r.GetResourceName())
+}

src/replication/ng/operation/controller.go

@@ -104,8 +104,9 @@ func (c *controller) createFlow(executionID int64, policy *model.Policy, resourc
 				Value: resource.Type,
 			},
 			{
-				Type:  model.FilterTypeName,
+				Type: model.FilterTypeName,
-				Value: resource.Metadata.Name,
+				// TODO only filter the repo part?
+				Value: resource.Metadata.GetResourceName(),
 			},
 			{
 				Type: model.FilterTypeTag,

src/replication/ng/operation/controller_test.go

@@ -132,7 +132,18 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
 func (f *fakedAdapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) {
 	return nil, nil
 }
-func (f *fakedAdapter) CreateNamespace(*model.Namespace) error {
+func (f *fakedAdapter) ConvertResourceMetadata(*model.ResourceMetadata, *model.Namespace) (*model.ResourceMetadata, error) {
+	return &model.ResourceMetadata{
+		Namespace: &model.Namespace{
+			Name: "library",
+		},
+		Repository: &model.Repository{
+			Name: "hello-world",
+		},
+		Vtags: []string{"latest"},
+	}, nil
+}
+func (f *fakedAdapter) PrepareForPush(*model.Resource) error {
 	return nil
 }
 func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {
@@ -155,9 +166,13 @@ func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter)
 		{
 			Type: model.ResourceTypeRepository,
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/hello-world",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{"latest"},
+				},
+				Repository: &model.Repository{
+					Name: "hello-world",
+				},
+				Vtags: []string{"latest"},
 			},
 			Override: false,
 		},
@@ -190,9 +205,13 @@ func (f *fakedAdapter) FetchCharts(namespaces []string, filters []*model.Filter)
 		{
 			Type: model.ResourceTypeChart,
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/harbor",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{"0.2.0"},
+				},
+				Repository: &model.Repository{
+					Name: "harbor",
+				},
+				Vtags: []string{"0.2.0"},
 			},
 		},
 	}, nil
@@ -232,7 +251,12 @@ func TestStartReplication(t *testing.T) {
 	resource := &model.Resource{
 		Type: model.ResourceTypeRepository,
 		Metadata: &model.ResourceMetadata{
-			Name:  "library/hello-world",
+			Namespace: &model.Namespace{
+				Name: "library",
+			},
+			Repository: &model.Repository{
+				Name: "hello-world",
+			},
 			Vtags: []string{"1.0", "2.0"},
 		},
 	}

src/replication/ng/operation/flow/copy.go

@@ -57,14 +57,13 @@ func (c *copyFlow) Run(interface{}) (int, error) {
 		log.Infof("no resources need to be replicated for the execution %d, skip", c.executionID)
 		return 0, nil
 	}
-	dstNamespaces, err := assembleDestinationNamespaces(srcAdapter, srcResources, c.policy.DestNamespace)
+	dstResources, err := assembleDestinationResources(dstAdapter, srcResources, c.policy)
 	if err != nil {
 		return 0, err
 	}
-	if err = createNamespaces(dstAdapter, dstNamespaces); err != nil {
+	if err = prepareForPush(dstAdapter, dstResources); err != nil {
 		return 0, err
 	}
-	dstResources := assembleDestinationResources(srcResources, c.policy.DestRegistry, c.policy.DestNamespace, c.policy.Override)
 	items, err := preprocess(c.scheduler, srcResources, dstResources)
 	if err != nil {
 		return 0, err

src/replication/ng/operation/flow/deletion.go

@@ -43,6 +43,10 @@ func NewDeletionFlow(executionMgr execution.Manager, scheduler scheduler.Schedul
 }
 
 func (d *deletionFlow) Run(interface{}) (int, error) {
+	_, dstAdapter, err := initialize(d.policy)
+	if err != nil {
+		return 0, err
+	}
 	// filling the registry information
 	for _, resource := range d.resources {
 		resource.Registry = d.policy.SrcRegistry
@@ -56,7 +60,10 @@ func (d *deletionFlow) Run(interface{}) (int, error) {
 		log.Infof("no resources need to be replicated for the execution %d, skip", d.executionID)
 		return 0, nil
 	}
-	dstResources := assembleDestinationResources(srcResources, d.policy.DestRegistry, d.policy.DestNamespace, d.policy.Override)
+	dstResources, err := assembleDestinationResources(dstAdapter, srcResources, d.policy)
+	if err != nil {
+		return 0, err
+	}
 	items, err := preprocess(d.scheduler, srcResources, dstResources)
 	if err != nil {
 		return 0, err

src/replication/ng/operation/flow/deletion_test.go

@@ -37,9 +37,13 @@ func TestRunOfDeletionFlow(t *testing.T) {
 	resources := []*model.Resource{
 		{
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/hello-world",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{"latest"},
+				},
+				Repository: &model.Repository{
+					Name: "hello-world",
+				},
+				Vtags: []string{"latest"},
 			},
 		},
 	}

src/replication/ng/operation/flow/stage.go

@@ -134,7 +134,8 @@ func filterResources(resources []*model.Resource, filters []*model.Filter) ([]*m
 					match = false
 					break
 				}
-				m, err := util.Match(pattern, resource.Metadata.Name)
+				// TODO filter only the repository part?
+				m, err := util.Match(pattern, resource.Metadata.GetResourceName())
 				if err != nil {
 					return nil, err
 				}
@@ -181,73 +182,46 @@ func filterResources(resources []*model.Resource, filters []*model.Filter) ([]*m
 	return res, nil
 }
 
-// Assemble the namespaces that need to be created on the destination registry:
+// assemble the destination resources by filling the metadata, registry and override properties
-// step 1: get the detail information for each of the source namespaces
+func assembleDestinationResources(adapter adp.Adapter, resources []*model.Resource,
-// step 2: if the destination namespace isn't specified in the policy, then the
+	policy *model.Policy) ([]*model.Resource, error) {
-// same namespaces with the source will be returned. If it is specified, then
-// returns the specified one with the merged metadatas of all source namespaces
-func assembleDestinationNamespaces(srcAdapter adp.Adapter, srcResources []*model.Resource, dstNamespace string) ([]*model.Namespace, error) {
-	namespaces := []*model.Namespace{}
-	for _, srcResource := range srcResources {
-		namespace, err := srcAdapter.GetNamespace(srcResource.Metadata.Namespace)
-		if err != nil {
-			return nil, err
-		}
-		namespaces = append(namespaces, namespace)
-	}
-
-	if len(dstNamespace) != 0 {
-		namespaces = []*model.Namespace{
-			{
-				Name: dstNamespace,
-				// TODO merge the metadata
-				Metadata: map[string]interface{}{},
-			},
-		}
-	}
-
-	log.Debug("assemble the destination namespaces completed")
-	return namespaces, nil
-}
-
-// create the namespaces on the destination registry
-func createNamespaces(adapter adp.Adapter, namespaces []*model.Namespace) error {
-	for _, namespace := range namespaces {
-		if err := adapter.CreateNamespace(namespace); err != nil {
-			return fmt.Errorf("failed to create the namespace %s on the destination registry: %v", namespace.Name, err)
-		}
-		log.Debugf("namespace %s created on the destination registry", namespace.Name)
-	}
-	return nil
-}
-
-// assemble the destination resources by filling the registry, namespace and override properties
-func assembleDestinationResources(resources []*model.Resource,
-	registry *model.Registry, namespace string, override bool) []*model.Resource {
 	result := []*model.Resource{}
+	var namespace *model.Namespace
+	if len(policy.DestNamespace) > 0 {
+		namespace = &model.Namespace{
+			Name: policy.DestNamespace,
+		}
+	}
 	for _, resource := range resources {
+		metadata, err := adapter.ConvertResourceMetadata(resource.Metadata, namespace)
+		if err != nil {
+			return nil, fmt.Errorf("failed to convert the resource metadata of %s: %v", resource.Metadata.GetResourceName(), err)
+		}
 		res := &model.Resource{
-			Type: resource.Type,
+			Type:         resource.Type,
-			Metadata: &model.ResourceMetadata{
+			Metadata:     metadata,
-				Name:      resource.Metadata.Name,
+			Registry:     policy.DestRegistry,
-				Namespace: resource.Metadata.Namespace,
-				Vtags:     resource.Metadata.Vtags,
-			},
-			Registry:     registry,
 			ExtendedInfo: resource.ExtendedInfo,
 			Deleted:      resource.Deleted,
-			Override:     override,
+			Override:     policy.Override,
-		}
-		// if the destination namespace is specified, use the specified one
-		if len(namespace) > 0 {
-			res.Metadata.Name = strings.Replace(resource.Metadata.Name,
-				resource.Metadata.Namespace, namespace, 1)
-			res.Metadata.Namespace = namespace
 		}
 		result = append(result, res)
 	}
 	log.Debug("assemble the destination resources completed")
-	return result
+	return result, nil
+}
+
+// do the prepare work for pushing/uploading the resources: create the namespace or repository
+func prepareForPush(adapter adp.Adapter, resources []*model.Resource) error {
+	// TODO need to consider how to handle that both contains public/private namespace
+	for _, resource := range resources {
+		name := resource.Metadata.GetResourceName()
+		if err := adapter.PrepareForPush(resource); err != nil {
+			return fmt.Errorf("failed to do the prepare work for pushing/uploading %s: %v", name, err)
+		}
+		log.Debugf("the prepare work for pushing/uploading %s completed", name)
+	}
+	return nil
 }
 
 // preprocess
@@ -341,7 +315,7 @@ func getResourceName(res *model.Resource) string {
 		return ""
 	}
 	if len(meta.Vtags) == 0 {
-		return meta.Name
+		return meta.GetResourceName()
 	}
-	return meta.Name + ":[" + strings.Join(meta.Vtags, ",") + "]"
+	return meta.GetResourceName() + ":[" + strings.Join(meta.Vtags, ",") + "]"
 }

src/replication/ng/operation/flow/stage_test.go

@@ -49,7 +49,14 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
 func (f *fakedAdapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) {
 	return nil, nil
 }
-func (f *fakedAdapter) CreateNamespace(*model.Namespace) error {
+func (f *fakedAdapter) ConvertResourceMetadata(metadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) {
+	if namespace != nil {
+		metadata.Namespace = namespace
+	}
+	return metadata, nil
+}
+
+func (f *fakedAdapter) PrepareForPush(*model.Resource) error {
 	return nil
 }
 func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {
@@ -72,9 +79,13 @@ func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter)
 		{
 			Type: model.ResourceTypeRepository,
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/hello-world",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{"latest"},
+				},
+				Repository: &model.Repository{
+					Name: "hello-world",
+				},
+				Vtags: []string{"latest"},
 			},
 			Override: false,
 		},
@@ -107,9 +118,13 @@ func (f *fakedAdapter) FetchCharts(namespaces []string, filters []*model.Filter)
 		{
 			Type: model.ResourceTypeChart,
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/harbor",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{"0.2.0"},
+				},
+				Repository: &model.Repository{
+					Name: "harbor",
+				},
+				Vtags: []string{"0.2.0"},
 			},
 		},
 	}, nil
@@ -226,9 +241,13 @@ func TestFilterResources(t *testing.T) {
 		{
 			Type: model.ResourceTypeRepository,
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/hello-world",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{"latest"},
+				},
+				Repository: &model.Repository{
+					Name: "hello-world",
+				},
+				Vtags: []string{"latest"},
 				// TODO test labels
 				Labels: nil,
 			},
@@ -238,9 +257,13 @@ func TestFilterResources(t *testing.T) {
 		{
 			Type: model.ResourceTypeChart,
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/harbor",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{"0.2.0", "0.3.0"},
+				},
+				Repository: &model.Repository{
+					Name: "harbor",
+				},
+				Vtags: []string{"0.2.0", "0.3.0"},
 				// TODO test labels
 				Labels: nil,
 			},
@@ -250,9 +273,13 @@ func TestFilterResources(t *testing.T) {
 		{
 			Type: model.ResourceTypeChart,
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/mysql",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{"1.0"},
+				},
+				Repository: &model.Repository{
+					Name: "mysql",
+				},
+				Vtags: []string{"1.0"},
 				// TODO test labels
 				Labels: nil,
 			},
@@ -281,67 +308,40 @@ func TestFilterResources(t *testing.T) {
 	res, err := filterResources(resources, filters)
 	require.Nil(t, err)
 	assert.Equal(t, 1, len(res))
-	assert.Equal(t, "library/harbor", res[0].Metadata.Name)
+	assert.Equal(t, "library", res[0].Metadata.Namespace.Name)
+	assert.Equal(t, "harbor", res[0].Metadata.Repository.Name)
 	assert.Equal(t, 1, len(res[0].Metadata.Vtags))
 	assert.Equal(t, "0.2.0", res[0].Metadata.Vtags[0])
 }
 
-func TestAssembleDestinationNamespaces(t *testing.T) {
-	adapter := &fakedAdapter{}
-	resources := []*model.Resource{
-		{
-			Metadata: &model.ResourceMetadata{
-				Namespace: "library",
-			},
-		},
-	}
-	namespace := ""
-	ns, err := assembleDestinationNamespaces(adapter, resources, namespace)
-	require.Nil(t, err)
-	assert.Equal(t, 1, len(ns))
-	assert.Equal(t, "library", ns[0].Name)
-	assert.Equal(t, true, ns[0].Metadata["public"].(bool))
-
-	namespace = "test"
-	ns, err = assembleDestinationNamespaces(adapter, resources, namespace)
-	require.Nil(t, err)
-	assert.Equal(t, 1, len(ns))
-	assert.Equal(t, "test", ns[0].Name)
-	// TODO add test for merged metadata
-	// assert.Equal(t, true, ns[0].Metadata["public"].(bool))
-}
-
-func TestCreateNamespaces(t *testing.T) {
-	adapter := &fakedAdapter{}
-	namespaces := []*model.Namespace{
-		{
-			Name: "library",
-		},
-	}
-	err := createNamespaces(adapter, namespaces)
-	require.Nil(t, err)
-}
-
 func TestAssembleDestinationResources(t *testing.T) {
+	adapter := &fakedAdapter{}
 	resources := []*model.Resource{
 		{
 			Type: model.ResourceTypeChart,
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/hello-world",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{"latest"},
+				},
+				Repository: &model.Repository{
+					Name: "hello-world",
+				},
+				Vtags: []string{"latest"},
 			},
 			Override: false,
 		},
 	}
-	registry := &model.Registry{}
+	policy := &model.Policy{
-	namespace := "test"
+		DestRegistry:  &model.Registry{},
-	override := true
+		DestNamespace: "test",
-	res := assembleDestinationResources(resources, registry, namespace, override)
+		Override:      true,
+	}
+	res, err := assembleDestinationResources(adapter, resources, policy)
+	require.Nil(t, err)
 	assert.Equal(t, 1, len(res))
 	assert.Equal(t, model.ResourceTypeChart, res[0].Type)
-	assert.Equal(t, "test/hello-world", res[0].Metadata.Name)
+	assert.Equal(t, "hello-world", res[0].Metadata.Repository.Name)
-	assert.Equal(t, namespace, res[0].Metadata.Namespace)
+	assert.Equal(t, "test", res[0].Metadata.Namespace.Name)
 	assert.Equal(t, 1, len(res[0].Metadata.Vtags))
 	assert.Equal(t, "latest", res[0].Metadata.Vtags[0])
 }
@@ -352,9 +352,13 @@ func TestPreprocess(t *testing.T) {
 		{
 			Type: model.ResourceTypeChart,
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/hello-world",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{"latest"},
+				},
+				Repository: &model.Repository{
+					Name: "hello-world",
+				},
+				Vtags: []string{"latest"},
 			},
 			Override: false,
 		},
@@ -363,9 +367,13 @@ func TestPreprocess(t *testing.T) {
 		{
 			Type: model.ResourceTypeChart,
 			Metadata: &model.ResourceMetadata{
-				Name:      "test/hello-world",
+				Namespace: &model.Namespace{
-				Namespace: "test",
+					Name: "test",
-				Vtags:     []string{"latest"},
+				},
+				Repository: &model.Repository{
+					Name: "hello-world",
+				},
+				Vtags: []string{"latest"},
 			},
 			Override: false,
 		},

src/replication/ng/operation/scheduler/scheduler_test.go

@@ -50,9 +50,11 @@ func TestStop(t *testing.T) {
 func generateData() ([]*ScheduleItem, error) {
 	srcResource := &model.Resource{
 		Metadata: &model.ResourceMetadata{
-			Namespace: "namespace1",
+			Namespace: &model.Namespace{
-			Vtags:     []string{"latest"},
+				Name: "namespace1",
-			Labels:    []string{"latest"},
+			},
+			Vtags:  []string{"latest"},
+			Labels: []string{"latest"},
 		},
 		Registry: &model.Registry{
 			Credential: &model.Credential{},
@@ -60,9 +62,11 @@ func generateData() ([]*ScheduleItem, error) {
 	}
 	destResource := &model.Resource{
 		Metadata: &model.ResourceMetadata{
-			Namespace: "namespace2",
+			Namespace: &model.Namespace{
-			Vtags:     []string{"v1", "v2"},
+				Name: "namespace2",
-			Labels:    []string{"latest"},
+			},
+			Vtags:  []string{"v1", "v2"},
+			Labels: []string{"latest"},
 		},
 		Registry: &model.Registry{
 			Credential: &model.Credential{},

src/replication/ng/transfer/chart/transfer.go

@@ -17,10 +17,9 @@ package chart
 import (
 	"errors"
 
-	"github.com/goharbor/harbor/src/replication/ng/adapter"
-
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/jobservice/errs"
+	"github.com/goharbor/harbor/src/replication/ng/adapter"
 	"github.com/goharbor/harbor/src/replication/ng/model"
 	trans "github.com/goharbor/harbor/src/replication/ng/transfer"
 )
@@ -63,17 +62,17 @@ func (t *transfer) Transfer(src *model.Resource, dst *model.Resource) error {
 	// delete the chart on destination registry
 	if dst.Deleted {
 		return t.delete(&chart{
-			name:    dst.Metadata.Name,
+			name:    dst.Metadata.GetResourceName(),
 			version: dst.Metadata.Vtags[0],
 		})
 	}
 
 	srcChart := &chart{
-		name:    src.Metadata.Name,
+		name:    src.Metadata.GetResourceName(),
 		version: src.Metadata.Vtags[0],
 	}
 	dstChart := &chart{
-		name:    dst.Metadata.Name,
+		name:    dst.Metadata.GetResourceName(),
 		version: dst.Metadata.Vtags[0],
 	}
 	// copy the chart from source registry to the destination

src/replication/ng/transfer/chart/transfer_test.go

@@ -34,9 +34,13 @@ func (f *fakeRegistry) FetchCharts(namespaces []string, filters []*model.Filter)
 		{
 			Type: model.ResourceTypeChart,
 			Metadata: &model.ResourceMetadata{
-				Name:      "library/harbor",
+				Namespace: &model.Namespace{
-				Namespace: "library",
+					Name: "library",
-				Vtags:     []string{"0.2.0"},
+				},
+				Repository: &model.Repository{
+					Name: "harbor",
+				},
+				Vtags: []string{"0.2.0"},
 			},
 		},
 	}, nil

src/replication/ng/transfer/repository/transfer.go

@@ -18,13 +18,12 @@ import (
 	"errors"
 	"strings"
 
-	"github.com/goharbor/harbor/src/replication/ng/adapter"
-
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/jobservice/errs"
+	"github.com/goharbor/harbor/src/replication/ng/adapter"
 	"github.com/goharbor/harbor/src/replication/ng/model"
 	trans "github.com/goharbor/harbor/src/replication/ng/transfer"
 )
@@ -67,17 +66,17 @@ func (t *transfer) Transfer(src *model.Resource, dst *model.Resource) error {
 	// delete the repository on destination registry
 	if dst.Deleted {
 		return t.delete(&repository{
-			repository: dst.Metadata.Name,
+			repository: dst.Metadata.GetResourceName(),
 			tags:       dst.Metadata.Vtags,
 		})
 	}
 
 	srcRepo := &repository{
-		repository: src.Metadata.Name,
+		repository: src.Metadata.GetResourceName(),
 		tags:       src.Metadata.Vtags,
 	}
 	dstRepo := &repository{
-		repository: dst.Metadata.Name,
+		repository: dst.Metadata.GetResourceName(),
 		tags:       dst.Metadata.Vtags,
 	}
 	// copy the repository from source registry to the destination