Merge pull request #7403 from ywk253100/190416_prepare_for_push

Update the PrepareForPush method

src/replication/adapter/adapter.go

@@ -30,9 +30,9 @@ type Factory func(*model.Registry) (Adapter, error)
 type Adapter interface {
 	// Info return the information of this adapter
 	Info() (*model.RegistryInfo, error)
-	// PrepareForPush does the prepare work that needed for pushing/uploading the resource
+	// PrepareForPush does the prepare work that needed for pushing/uploading the resources
 	// eg: create the namespace or repository
-	PrepareForPush(*model.Resource) error
+	PrepareForPush([]*model.Resource) error
 	// HealthCheck checks health status of registry
 	HealthCheck() (model.HealthStatus, error)
 }

src/replication/adapter/dockerhub/adapter.go

@@ -79,7 +79,9 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
 
 // PrepareForPush does the prepare work that needed for pushing/uploading the resource
 // eg: create the namespace or repository
-func (a *adapter) PrepareForPush(resource *model.Resource) error {
+func (a *adapter) PrepareForPush(resources []*model.Resource) error {
+	namespaces := map[string]struct{}{}
+	for _, resource := range resources {
 		if resource == nil {
 			return errors.New("the resource cannot be null")
 		}
@@ -97,16 +99,20 @@ func (a *adapter) PrepareForPush(resource *model.Resource) error {
 		// just skip here and the following task will fail
 		if len(namespace) == 0 {
 			log.Debug("the namespace is empty, skip")
-		return nil
+			continue
 		}
 
+		namespaces[namespace] = struct{}{}
+	}
+
+	for namespace := range namespaces {
 		err := a.CreateNamespace(&model.Namespace{
 			Name: namespace,
 		})
 		if err != nil {
 			return fmt.Errorf("create namespace '%s' in DockerHub error: %v", namespace, err)
 		}
-
+	}
 	return nil
 }
 

src/replication/adapter/harbor/adapter.go

@@ -136,7 +136,9 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
 	return info, nil
 }
 
-func (a *adapter) PrepareForPush(resource *model.Resource) error {
+func (a *adapter) PrepareForPush(resources []*model.Resource) error {
+	projects := map[string]*project{}
+	for _, resource := range resources {
 		if resource == nil {
 			return errors.New("the resource cannot be null")
 		}
@@ -154,15 +156,22 @@ func (a *adapter) PrepareForPush(resource *model.Resource) error {
 		// just skip here and the following task will fail
 		if len(projectName) == 0 {
 			log.Debug("the project name is empty, skip")
+			continue
+		}
+		// TODO handle the public
+		projects[projectName] = &project{
+			Name: projectName,
+		}
+	}
+	for _, project := range projects {
+		err := a.client.Post(a.coreServiceURL+"/api/projects", project)
+		if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
+			log.Debugf("got 409 when trying to create project %s", project.Name)
 			return nil
 		}
-	project := &struct {
+		return err
-		Name     string                 `json:"project_name"`
-		Metadata map[string]interface{} `json:"metadata"`
-	}{
-		Name: projectName,
-		// TODO handle the public
 	}
+	return nil
 
 	// TODO
 	/*
@@ -185,13 +194,6 @@ func (a *adapter) PrepareForPush(resource *model.Resource) error {
 			}
 		}
 	*/
-
-	err := a.client.Post(a.coreServiceURL+"/api/projects", project)
-	if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
-		log.Debugf("got 409 when trying to create project %s", projectName)
-		return nil
-	}
-	return err
 }
 
 type project struct {

src/replication/adapter/harbor/adapter_test.go

@@ -87,30 +87,41 @@ func TestPrepareForPush(t *testing.T) {
 	adapter, err := newAdapter(registry)
 	require.Nil(t, err)
 	// nil resource
-	err = adapter.PrepareForPush(nil)
+	err = adapter.PrepareForPush([]*model.Resource{nil})
 	require.NotNil(t, err)
 	// nil metadata
-	err = adapter.PrepareForPush(&model.Resource{})
+	err = adapter.PrepareForPush([]*model.Resource{
+		{},
+	})
 	require.NotNil(t, err)
 	// nil repository
-	err = adapter.PrepareForPush(&model.Resource{
+	err = adapter.PrepareForPush(
+		[]*model.Resource{
+			{
 				Metadata: &model.ResourceMetadata{},
+			},
 		})
 	require.NotNil(t, err)
 	// nil repository name
-	err = adapter.PrepareForPush(&model.Resource{
+	err = adapter.PrepareForPush(
+		[]*model.Resource{
+			{
 				Metadata: &model.ResourceMetadata{
 					Repository: &model.Repository{},
 				},
+			},
 		})
 	require.NotNil(t, err)
 	// project doesn't exist
-	err = adapter.PrepareForPush(&model.Resource{
+	err = adapter.PrepareForPush(
+		[]*model.Resource{
+			{
 				Metadata: &model.ResourceMetadata{
 					Repository: &model.Repository{
 						Name: "library/hello-world",
 					},
 				},
+			},
 		})
 	require.Nil(t, err)
 
@@ -129,12 +140,15 @@ func TestPrepareForPush(t *testing.T) {
 	}
 	adapter, err = newAdapter(registry)
 	require.Nil(t, err)
-	err = adapter.PrepareForPush(&model.Resource{
+	err = adapter.PrepareForPush(
+		[]*model.Resource{
+			{
 				Metadata: &model.ResourceMetadata{
 					Repository: &model.Repository{
 						Name: "library/hello-world",
 					},
 				},
+			},
 		})
 	require.Nil(t, err)
 }

src/replication/adapter/huawei/huawei_adapter.go

@@ -120,8 +120,9 @@ func (adapter Adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceM
 }
 
 // PrepareForPush prepare for push to Huawei SWR
-func (adapter Adapter) PrepareForPush(resource *model.Resource) error {
+func (adapter Adapter) PrepareForPush(resources []*model.Resource) error {
-
+	// TODO optimize the logic by merging the same namesapces
+	for _, resource := range resources {
 		namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name)
 		ns, err := adapter.GetNamespace(namespace)
 		if err != nil {
@@ -168,6 +169,7 @@ func (adapter Adapter) PrepareForPush(resource *model.Resource) error {
 			body, _ := ioutil.ReadAll(resp.Body)
 			return fmt.Errorf("[%d][%s]", code, string(body))
 		}
+	}
 	return nil
 }
 

src/replication/adapter/huawei/huawei_adapter_test.go

@@ -49,7 +49,7 @@ func TestAdapter_PrepareForPush(t *testing.T) {
 		Repository: repository,
 	}
 	resource.Metadata = metadata
-	err := hwAdapter.PrepareForPush(resource)
+	err := hwAdapter.PrepareForPush([]*model.Resource{resource})
 	if err != nil {
 		if strings.HasPrefix(err.Error(), "[401]") {
 			t.Log("huawei ak/sk is not available", err.Error())

src/replication/adapter/native/adapter.go

@@ -74,4 +74,4 @@ func (native) Info() (info *model.RegistryInfo, err error) {
 }
 
 // PrepareForPush nothing need to do.
-func (native) PrepareForPush(*model.Resource) error { return nil }
+func (native) PrepareForPush([]*model.Resource) error { return nil }

src/replication/operation/controller_test.go

@@ -130,7 +130,7 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
 	}, nil
 }
 
-func (f *fakedAdapter) PrepareForPush(*model.Resource) error {
+func (f *fakedAdapter) PrepareForPush([]*model.Resource) error {
 	return nil
 }
 func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {

src/replication/operation/flow/stage.go

@@ -209,14 +209,10 @@ func assembleDestinationResources(resources []*model.Resource,
 
 // do the prepare work for pushing/uploading the resources: create the namespace or repository
 func prepareForPush(adapter adp.Adapter, resources []*model.Resource) error {
-	// TODO need to consider how to handle that both contains public/private namespace
+	if err := adapter.PrepareForPush(resources); err != nil {
-	for _, resource := range resources {
+		return fmt.Errorf("failed to do the prepare work for pushing/uploading resources: %v", err)
-		name := resource.Metadata.Repository.Name
-		if err := adapter.PrepareForPush(resource); err != nil {
-			return fmt.Errorf("failed to do the prepare work for pushing/uploading %s: %v", name, err)
-		}
-		log.Debugf("the prepare work for pushing/uploading %s completed", name)
 	}
+	log.Debug("the prepare work for pushing/uploading resources completed")
 	return nil
 }
 

src/replication/operation/flow/stage_test.go

@@ -46,7 +46,7 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
 	}, nil
 }
 
-func (f *fakedAdapter) PrepareForPush(*model.Resource) error {
+func (f *fakedAdapter) PrepareForPush([]*model.Resource) error {
 	return nil
 }
 func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {