Update the PrepareForPush method

Update the PrepareForPush method to avoid the useless API callings to create namespaces

Signed-off-by: Wenkai Yin <yinw@vmware.com>

src/replication/adapter/adapter.go

@@ -30,9 +30,9 @@ type Factory func(*model.Registry) (Adapter, error)
 type Adapter interface {
 	// Info return the information of this adapter
 	Info() (*model.RegistryInfo, error)
-	// PrepareForPush does the prepare work that needed for pushing/uploading the resource
+	// PrepareForPush does the prepare work that needed for pushing/uploading the resources
 	// eg: create the namespace or repository
-	PrepareForPush(*model.Resource) error
+	PrepareForPush([]*model.Resource) error
 	// HealthCheck checks health status of registry
 	HealthCheck() (model.HealthStatus, error)
 }

src/replication/adapter/dockerhub/adapter.go

@@ -79,34 +79,40 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
 
 // PrepareForPush does the prepare work that needed for pushing/uploading the resource
 // eg: create the namespace or repository
-func (a *adapter) PrepareForPush(resource *model.Resource) error {
+func (a *adapter) PrepareForPush(resources []*model.Resource) error {
-	if resource == nil {
+	namespaces := map[string]struct{}{}
-		return errors.New("the resource cannot be null")
+	for _, resource := range resources {
-	}
+		if resource == nil {
-	if resource.Metadata == nil {
+			return errors.New("the resource cannot be null")
-		return errors.New("the metadata of resource cannot be null")
+		}
-	}
+		if resource.Metadata == nil {
-	if resource.Metadata.Repository == nil {
+			return errors.New("the metadata of resource cannot be null")
-		return errors.New("the namespace of resource cannot be null")
+		}
-	}
+		if resource.Metadata.Repository == nil {
-	if len(resource.Metadata.Repository.Name) == 0 {
+			return errors.New("the namespace of resource cannot be null")
-		return errors.New("the name of the namespace cannot be null")
+		}
-	}
+		if len(resource.Metadata.Repository.Name) == 0 {
-	namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name)
+			return errors.New("the name of the namespace cannot be null")
-	// Docker Hub doesn't support the repository contains no "/"
+		}
-	// just skip here and the following task will fail
+		namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name)
-	if len(namespace) == 0 {
+		// Docker Hub doesn't support the repository contains no "/"
-		log.Debug("the namespace is empty, skip")
+		// just skip here and the following task will fail
-		return nil
+		if len(namespace) == 0 {
+			log.Debug("the namespace is empty, skip")
+			continue
+		}
+
+		namespaces[namespace] = struct{}{}
 	}
 
-	err := a.CreateNamespace(&model.Namespace{
+	for namespace := range namespaces {
-		Name: namespace,
+		err := a.CreateNamespace(&model.Namespace{
-	})
+			Name: namespace,
-	if err != nil {
+		})
-		return fmt.Errorf("create namespace '%s' in DockerHub error: %v", namespace, err)
+		if err != nil {
+			return fmt.Errorf("create namespace '%s' in DockerHub error: %v", namespace, err)
+		}
 	}
-
 	return nil
 }
 

src/replication/adapter/harbor/adapter.go

@@ -136,33 +136,42 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
 	return info, nil
 }
 
-func (a *adapter) PrepareForPush(resource *model.Resource) error {
+func (a *adapter) PrepareForPush(resources []*model.Resource) error {
-	if resource == nil {
+	projects := map[string]*project{}
-		return errors.New("the resource cannot be null")
+	for _, resource := range resources {
-	}
+		if resource == nil {
-	if resource.Metadata == nil {
+			return errors.New("the resource cannot be null")
-		return errors.New("the metadata of resource cannot be null")
+		}
-	}
+		if resource.Metadata == nil {
-	if resource.Metadata.Repository == nil {
+			return errors.New("the metadata of resource cannot be null")
-		return errors.New("the repository of resource cannot be null")
+		}
-	}
+		if resource.Metadata.Repository == nil {
-	if len(resource.Metadata.Repository.Name) == 0 {
+			return errors.New("the repository of resource cannot be null")
-		return errors.New("the name of the repository cannot be null")
+		}
-	}
+		if len(resource.Metadata.Repository.Name) == 0 {
-	projectName, _ := util.ParseRepository(resource.Metadata.Repository.Name)
+			return errors.New("the name of the repository cannot be null")
-	// harbor doesn't support the repository contains no "/"
+		}
-	// just skip here and the following task will fail
+		projectName, _ := util.ParseRepository(resource.Metadata.Repository.Name)
-	if len(projectName) == 0 {
+		// harbor doesn't support the repository contains no "/"
-		log.Debug("the project name is empty, skip")
+		// just skip here and the following task will fail
-		return nil
+		if len(projectName) == 0 {
-	}
+			log.Debug("the project name is empty, skip")
-	project := &struct {
+			continue
-		Name     string                 `json:"project_name"`
+		}
-		Metadata map[string]interface{} `json:"metadata"`
-	}{
-		Name: projectName,
 		// TODO handle the public
+		projects[projectName] = &project{
+			Name: projectName,
+		}
 	}
+	for _, project := range projects {
+		err := a.client.Post(a.coreServiceURL+"/api/projects", project)
+		if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
+			log.Debugf("got 409 when trying to create project %s", project.Name)
+			return nil
+		}
+		return err
+	}
+	return nil
 
 	// TODO
 	/*
@@ -185,13 +194,6 @@ func (a *adapter) PrepareForPush(resource *model.Resource) error {
 			}
 		}
 	*/
-
-	err := a.client.Post(a.coreServiceURL+"/api/projects", project)
-	if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
-		log.Debugf("got 409 when trying to create project %s", projectName)
-		return nil
-	}
-	return err
 }
 
 type project struct {

src/replication/adapter/harbor/adapter_test.go

@@ -87,31 +87,42 @@ func TestPrepareForPush(t *testing.T) {
 	adapter, err := newAdapter(registry)
 	require.Nil(t, err)
 	// nil resource
-	err = adapter.PrepareForPush(nil)
+	err = adapter.PrepareForPush([]*model.Resource{nil})
 	require.NotNil(t, err)
 	// nil metadata
-	err = adapter.PrepareForPush(&model.Resource{})
+	err = adapter.PrepareForPush([]*model.Resource{
+		{},
+	})
 	require.NotNil(t, err)
 	// nil repository
-	err = adapter.PrepareForPush(&model.Resource{
+	err = adapter.PrepareForPush(
-		Metadata: &model.ResourceMetadata{},
+		[]*model.Resource{
-	})
+			{
+				Metadata: &model.ResourceMetadata{},
+			},
+		})
 	require.NotNil(t, err)
 	// nil repository name
-	err = adapter.PrepareForPush(&model.Resource{
+	err = adapter.PrepareForPush(
-		Metadata: &model.ResourceMetadata{
+		[]*model.Resource{
-			Repository: &model.Repository{},
+			{
-		},
+				Metadata: &model.ResourceMetadata{
-	})
+					Repository: &model.Repository{},
+				},
+			},
+		})
 	require.NotNil(t, err)
 	// project doesn't exist
-	err = adapter.PrepareForPush(&model.Resource{
+	err = adapter.PrepareForPush(
-		Metadata: &model.ResourceMetadata{
+		[]*model.Resource{
-			Repository: &model.Repository{
+			{
-				Name: "library/hello-world",
+				Metadata: &model.ResourceMetadata{
+					Repository: &model.Repository{
+						Name: "library/hello-world",
+					},
+				},
 			},
-		},
+		})
-	})
 	require.Nil(t, err)
 
 	server.Close()
@@ -129,12 +140,15 @@ func TestPrepareForPush(t *testing.T) {
 	}
 	adapter, err = newAdapter(registry)
 	require.Nil(t, err)
-	err = adapter.PrepareForPush(&model.Resource{
+	err = adapter.PrepareForPush(
-		Metadata: &model.ResourceMetadata{
+		[]*model.Resource{
-			Repository: &model.Repository{
+			{
-				Name: "library/hello-world",
+				Metadata: &model.ResourceMetadata{
+					Repository: &model.Repository{
+						Name: "library/hello-world",
+					},
+				},
 			},
-		},
+		})
-	})
 	require.Nil(t, err)
 }

src/replication/adapter/huawei/huawei_adapter.go

@@ -120,53 +120,55 @@ func (adapter Adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceM
 }
 
 // PrepareForPush prepare for push to Huawei SWR
-func (adapter Adapter) PrepareForPush(resource *model.Resource) error {
+func (adapter Adapter) PrepareForPush(resources []*model.Resource) error {
-
+	// TODO optimize the logic by merging the same namesapces
-	namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name)
+	for _, resource := range resources {
-	ns, err := adapter.GetNamespace(namespace)
+		namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name)
-	if err != nil {
+		ns, err := adapter.GetNamespace(namespace)
-		//
+		if err != nil {
-	} else {
+			//
-		if ns.Name == namespace {
+		} else {
-			return nil
+			if ns.Name == namespace {
+				return nil
+			}
 		}
-	}
 
-	url := fmt.Sprintf("%s/dockyard/v2/namespaces", adapter.Registry.URL)
+		url := fmt.Sprintf("%s/dockyard/v2/namespaces", adapter.Registry.URL)
-	namespacebyte, err := json.Marshal(struct {
+		namespacebyte, err := json.Marshal(struct {
-		Namespace string `json:"namespace"`
+			Namespace string `json:"namespace"`
-	}{Namespace: namespace})
+		}{Namespace: namespace})
-	if err != nil {
+		if err != nil {
-		return err
+			return err
-	}
-
-	r, err := http.NewRequest("POST", url, strings.NewReader(string(namespacebyte)))
-	if err != nil {
-		return err
-	}
-
-	r.Header.Add("content-type", "application/json; charset=utf-8")
-	encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret)))
-	r.Header.Add("Authorization", "Basic "+encodeAuth)
-
-	client := &http.Client{}
-	if adapter.Registry.Insecure == true {
-		client = &http.Client{
-			Transport: &http.Transport{
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-			},
 		}
-	}
-	resp, err := client.Do(r)
-	if err != nil {
-		return err
-	}
 
-	defer resp.Body.Close()
+		r, err := http.NewRequest("POST", url, strings.NewReader(string(namespacebyte)))
-	code := resp.StatusCode
+		if err != nil {
-	if code >= 300 || code < 200 {
+			return err
-		body, _ := ioutil.ReadAll(resp.Body)
+		}
-		return fmt.Errorf("[%d][%s]", code, string(body))
+
+		r.Header.Add("content-type", "application/json; charset=utf-8")
+		encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret)))
+		r.Header.Add("Authorization", "Basic "+encodeAuth)
+
+		client := &http.Client{}
+		if adapter.Registry.Insecure == true {
+			client = &http.Client{
+				Transport: &http.Transport{
+					TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+				},
+			}
+		}
+		resp, err := client.Do(r)
+		if err != nil {
+			return err
+		}
+
+		defer resp.Body.Close()
+		code := resp.StatusCode
+		if code >= 300 || code < 200 {
+			body, _ := ioutil.ReadAll(resp.Body)
+			return fmt.Errorf("[%d][%s]", code, string(body))
+		}
 	}
 	return nil
 }

src/replication/adapter/huawei/huawei_adapter_test.go

@@ -49,7 +49,7 @@ func TestAdapter_PrepareForPush(t *testing.T) {
 		Repository: repository,
 	}
 	resource.Metadata = metadata
-	err := hwAdapter.PrepareForPush(resource)
+	err := hwAdapter.PrepareForPush([]*model.Resource{resource})
 	if err != nil {
 		if strings.HasPrefix(err.Error(), "[401]") {
 			t.Log("huawei ak/sk is not available", err.Error())

src/replication/adapter/native/adapter.go

@@ -74,4 +74,4 @@ func (native) Info() (info *model.RegistryInfo, err error) {
 }
 
 // PrepareForPush nothing need to do.
-func (native) PrepareForPush(*model.Resource) error { return nil }
+func (native) PrepareForPush([]*model.Resource) error { return nil }

src/replication/operation/controller_test.go

@@ -130,7 +130,7 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
 	}, nil
 }
 
-func (f *fakedAdapter) PrepareForPush(*model.Resource) error {
+func (f *fakedAdapter) PrepareForPush([]*model.Resource) error {
 	return nil
 }
 func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {

src/replication/operation/flow/stage.go

@@ -209,14 +209,10 @@ func assembleDestinationResources(resources []*model.Resource,
 
 // do the prepare work for pushing/uploading the resources: create the namespace or repository
 func prepareForPush(adapter adp.Adapter, resources []*model.Resource) error {
-	// TODO need to consider how to handle that both contains public/private namespace
+	if err := adapter.PrepareForPush(resources); err != nil {
-	for _, resource := range resources {
+		return fmt.Errorf("failed to do the prepare work for pushing/uploading resources: %v", err)
-		name := resource.Metadata.Repository.Name
-		if err := adapter.PrepareForPush(resource); err != nil {
-			return fmt.Errorf("failed to do the prepare work for pushing/uploading %s: %v", name, err)
-		}
-		log.Debugf("the prepare work for pushing/uploading %s completed", name)
 	}
+	log.Debug("the prepare work for pushing/uploading resources completed")
 	return nil
 }
 

src/replication/operation/flow/stage_test.go

@@ -46,7 +46,7 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
 	}, nil
 }
 
-func (f *fakedAdapter) PrepareForPush(*model.Resource) error {
+func (f *fakedAdapter) PrepareForPush([]*model.Resource) error {
 	return nil
 }
 func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {