From 613dfee5eebcbdc6b4d079bbd7aba7566d71e86e Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 16 Apr 2019 16:40:31 +0800 Subject: [PATCH] Update the PrepareForPush method Update the PrepareForPush method to avoid the useless API callings to create namespaces Signed-off-by: Wenkai Yin --- src/replication/adapter/adapter.go | 4 +- src/replication/adapter/dockerhub/adapter.go | 56 ++++++------ src/replication/adapter/harbor/adapter.go | 66 +++++++------- .../adapter/harbor/adapter_test.go | 58 +++++++----- .../adapter/huawei/huawei_adapter.go | 88 ++++++++++--------- .../adapter/huawei/huawei_adapter_test.go | 2 +- src/replication/adapter/native/adapter.go | 2 +- src/replication/operation/controller_test.go | 2 +- src/replication/operation/flow/stage.go | 10 +-- src/replication/operation/flow/stage_test.go | 2 +- 10 files changed, 155 insertions(+), 135 deletions(-) diff --git a/src/replication/adapter/adapter.go b/src/replication/adapter/adapter.go index 5aba12704..ed502c347 100644 --- a/src/replication/adapter/adapter.go +++ b/src/replication/adapter/adapter.go @@ -30,9 +30,9 @@ type Factory func(*model.Registry) (Adapter, error) type Adapter interface { // Info return the information of this adapter Info() (*model.RegistryInfo, error) - // PrepareForPush does the prepare work that needed for pushing/uploading the resource + // PrepareForPush does the prepare work that needed for pushing/uploading the resources // eg: create the namespace or repository - PrepareForPush(*model.Resource) error + PrepareForPush([]*model.Resource) error // HealthCheck checks health status of registry HealthCheck() (model.HealthStatus, error) } diff --git a/src/replication/adapter/dockerhub/adapter.go b/src/replication/adapter/dockerhub/adapter.go index 39df186a3..9d5b27b7c 100644 --- a/src/replication/adapter/dockerhub/adapter.go +++ b/src/replication/adapter/dockerhub/adapter.go @@ -79,34 +79,40 @@ func (a *adapter) Info() (*model.RegistryInfo, error) { // PrepareForPush does the prepare work that needed for pushing/uploading the resource // eg: create the namespace or repository -func (a *adapter) PrepareForPush(resource *model.Resource) error { - if resource == nil { - return errors.New("the resource cannot be null") - } - if resource.Metadata == nil { - return errors.New("the metadata of resource cannot be null") - } - if resource.Metadata.Repository == nil { - return errors.New("the namespace of resource cannot be null") - } - if len(resource.Metadata.Repository.Name) == 0 { - return errors.New("the name of the namespace cannot be null") - } - namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name) - // Docker Hub doesn't support the repository contains no "/" - // just skip here and the following task will fail - if len(namespace) == 0 { - log.Debug("the namespace is empty, skip") - return nil +func (a *adapter) PrepareForPush(resources []*model.Resource) error { + namespaces := map[string]struct{}{} + for _, resource := range resources { + if resource == nil { + return errors.New("the resource cannot be null") + } + if resource.Metadata == nil { + return errors.New("the metadata of resource cannot be null") + } + if resource.Metadata.Repository == nil { + return errors.New("the namespace of resource cannot be null") + } + if len(resource.Metadata.Repository.Name) == 0 { + return errors.New("the name of the namespace cannot be null") + } + namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name) + // Docker Hub doesn't support the repository contains no "/" + // just skip here and the following task will fail + if len(namespace) == 0 { + log.Debug("the namespace is empty, skip") + continue + } + + namespaces[namespace] = struct{}{} } - err := a.CreateNamespace(&model.Namespace{ - Name: namespace, - }) - if err != nil { - return fmt.Errorf("create namespace '%s' in DockerHub error: %v", namespace, err) + for namespace := range namespaces { + err := a.CreateNamespace(&model.Namespace{ + Name: namespace, + }) + if err != nil { + return fmt.Errorf("create namespace '%s' in DockerHub error: %v", namespace, err) + } } - return nil } diff --git a/src/replication/adapter/harbor/adapter.go b/src/replication/adapter/harbor/adapter.go index fadc7bc9f..9b7157903 100644 --- a/src/replication/adapter/harbor/adapter.go +++ b/src/replication/adapter/harbor/adapter.go @@ -136,33 +136,42 @@ func (a *adapter) Info() (*model.RegistryInfo, error) { return info, nil } -func (a *adapter) PrepareForPush(resource *model.Resource) error { - if resource == nil { - return errors.New("the resource cannot be null") - } - if resource.Metadata == nil { - return errors.New("the metadata of resource cannot be null") - } - if resource.Metadata.Repository == nil { - return errors.New("the repository of resource cannot be null") - } - if len(resource.Metadata.Repository.Name) == 0 { - return errors.New("the name of the repository cannot be null") - } - projectName, _ := util.ParseRepository(resource.Metadata.Repository.Name) - // harbor doesn't support the repository contains no "/" - // just skip here and the following task will fail - if len(projectName) == 0 { - log.Debug("the project name is empty, skip") - return nil - } - project := &struct { - Name string `json:"project_name"` - Metadata map[string]interface{} `json:"metadata"` - }{ - Name: projectName, +func (a *adapter) PrepareForPush(resources []*model.Resource) error { + projects := map[string]*project{} + for _, resource := range resources { + if resource == nil { + return errors.New("the resource cannot be null") + } + if resource.Metadata == nil { + return errors.New("the metadata of resource cannot be null") + } + if resource.Metadata.Repository == nil { + return errors.New("the repository of resource cannot be null") + } + if len(resource.Metadata.Repository.Name) == 0 { + return errors.New("the name of the repository cannot be null") + } + projectName, _ := util.ParseRepository(resource.Metadata.Repository.Name) + // harbor doesn't support the repository contains no "/" + // just skip here and the following task will fail + if len(projectName) == 0 { + log.Debug("the project name is empty, skip") + continue + } // TODO handle the public + projects[projectName] = &project{ + Name: projectName, + } } + for _, project := range projects { + err := a.client.Post(a.coreServiceURL+"/api/projects", project) + if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict { + log.Debugf("got 409 when trying to create project %s", project.Name) + return nil + } + return err + } + return nil // TODO /* @@ -185,13 +194,6 @@ func (a *adapter) PrepareForPush(resource *model.Resource) error { } } */ - - err := a.client.Post(a.coreServiceURL+"/api/projects", project) - if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict { - log.Debugf("got 409 when trying to create project %s", projectName) - return nil - } - return err } type project struct { diff --git a/src/replication/adapter/harbor/adapter_test.go b/src/replication/adapter/harbor/adapter_test.go index d9ccab1c9..94e201d1d 100644 --- a/src/replication/adapter/harbor/adapter_test.go +++ b/src/replication/adapter/harbor/adapter_test.go @@ -87,31 +87,42 @@ func TestPrepareForPush(t *testing.T) { adapter, err := newAdapter(registry) require.Nil(t, err) // nil resource - err = adapter.PrepareForPush(nil) + err = adapter.PrepareForPush([]*model.Resource{nil}) require.NotNil(t, err) // nil metadata - err = adapter.PrepareForPush(&model.Resource{}) + err = adapter.PrepareForPush([]*model.Resource{ + {}, + }) require.NotNil(t, err) // nil repository - err = adapter.PrepareForPush(&model.Resource{ - Metadata: &model.ResourceMetadata{}, - }) + err = adapter.PrepareForPush( + []*model.Resource{ + { + Metadata: &model.ResourceMetadata{}, + }, + }) require.NotNil(t, err) // nil repository name - err = adapter.PrepareForPush(&model.Resource{ - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{}, - }, - }) + err = adapter.PrepareForPush( + []*model.Resource{ + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{}, + }, + }, + }) require.NotNil(t, err) // project doesn't exist - err = adapter.PrepareForPush(&model.Resource{ - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", + err = adapter.PrepareForPush( + []*model.Resource{ + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: "library/hello-world", + }, + }, }, - }, - }) + }) require.Nil(t, err) server.Close() @@ -129,12 +140,15 @@ func TestPrepareForPush(t *testing.T) { } adapter, err = newAdapter(registry) require.Nil(t, err) - err = adapter.PrepareForPush(&model.Resource{ - Metadata: &model.ResourceMetadata{ - Repository: &model.Repository{ - Name: "library/hello-world", + err = adapter.PrepareForPush( + []*model.Resource{ + { + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: "library/hello-world", + }, + }, }, - }, - }) + }) require.Nil(t, err) } diff --git a/src/replication/adapter/huawei/huawei_adapter.go b/src/replication/adapter/huawei/huawei_adapter.go index ef09d554d..7deca8b89 100644 --- a/src/replication/adapter/huawei/huawei_adapter.go +++ b/src/replication/adapter/huawei/huawei_adapter.go @@ -120,53 +120,55 @@ func (adapter Adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceM } // PrepareForPush prepare for push to Huawei SWR -func (adapter Adapter) PrepareForPush(resource *model.Resource) error { - - namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name) - ns, err := adapter.GetNamespace(namespace) - if err != nil { - // - } else { - if ns.Name == namespace { - return nil +func (adapter Adapter) PrepareForPush(resources []*model.Resource) error { + // TODO optimize the logic by merging the same namesapces + for _, resource := range resources { + namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name) + ns, err := adapter.GetNamespace(namespace) + if err != nil { + // + } else { + if ns.Name == namespace { + return nil + } } - } - url := fmt.Sprintf("%s/dockyard/v2/namespaces", adapter.Registry.URL) - namespacebyte, err := json.Marshal(struct { - Namespace string `json:"namespace"` - }{Namespace: namespace}) - if err != nil { - return err - } - - r, err := http.NewRequest("POST", url, strings.NewReader(string(namespacebyte))) - if err != nil { - return err - } - - r.Header.Add("content-type", "application/json; charset=utf-8") - encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret))) - r.Header.Add("Authorization", "Basic "+encodeAuth) - - client := &http.Client{} - if adapter.Registry.Insecure == true { - client = &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, + url := fmt.Sprintf("%s/dockyard/v2/namespaces", adapter.Registry.URL) + namespacebyte, err := json.Marshal(struct { + Namespace string `json:"namespace"` + }{Namespace: namespace}) + if err != nil { + return err } - } - resp, err := client.Do(r) - if err != nil { - return err - } - defer resp.Body.Close() - code := resp.StatusCode - if code >= 300 || code < 200 { - body, _ := ioutil.ReadAll(resp.Body) - return fmt.Errorf("[%d][%s]", code, string(body)) + r, err := http.NewRequest("POST", url, strings.NewReader(string(namespacebyte))) + if err != nil { + return err + } + + r.Header.Add("content-type", "application/json; charset=utf-8") + encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret))) + r.Header.Add("Authorization", "Basic "+encodeAuth) + + client := &http.Client{} + if adapter.Registry.Insecure == true { + client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + } + resp, err := client.Do(r) + if err != nil { + return err + } + + defer resp.Body.Close() + code := resp.StatusCode + if code >= 300 || code < 200 { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("[%d][%s]", code, string(body)) + } } return nil } diff --git a/src/replication/adapter/huawei/huawei_adapter_test.go b/src/replication/adapter/huawei/huawei_adapter_test.go index d03061fe0..db6eae94a 100644 --- a/src/replication/adapter/huawei/huawei_adapter_test.go +++ b/src/replication/adapter/huawei/huawei_adapter_test.go @@ -49,7 +49,7 @@ func TestAdapter_PrepareForPush(t *testing.T) { Repository: repository, } resource.Metadata = metadata - err := hwAdapter.PrepareForPush(resource) + err := hwAdapter.PrepareForPush([]*model.Resource{resource}) if err != nil { if strings.HasPrefix(err.Error(), "[401]") { t.Log("huawei ak/sk is not available", err.Error()) diff --git a/src/replication/adapter/native/adapter.go b/src/replication/adapter/native/adapter.go index ad9626b87..e4ffbbed5 100644 --- a/src/replication/adapter/native/adapter.go +++ b/src/replication/adapter/native/adapter.go @@ -74,4 +74,4 @@ func (native) Info() (info *model.RegistryInfo, err error) { } // PrepareForPush nothing need to do. -func (native) PrepareForPush(*model.Resource) error { return nil } +func (native) PrepareForPush([]*model.Resource) error { return nil } diff --git a/src/replication/operation/controller_test.go b/src/replication/operation/controller_test.go index 1c8448fbb..b379010c2 100644 --- a/src/replication/operation/controller_test.go +++ b/src/replication/operation/controller_test.go @@ -130,7 +130,7 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) { }, nil } -func (f *fakedAdapter) PrepareForPush(*model.Resource) error { +func (f *fakedAdapter) PrepareForPush([]*model.Resource) error { return nil } func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) { diff --git a/src/replication/operation/flow/stage.go b/src/replication/operation/flow/stage.go index 7fefba0da..268699bf9 100644 --- a/src/replication/operation/flow/stage.go +++ b/src/replication/operation/flow/stage.go @@ -209,14 +209,10 @@ func assembleDestinationResources(resources []*model.Resource, // do the prepare work for pushing/uploading the resources: create the namespace or repository func prepareForPush(adapter adp.Adapter, resources []*model.Resource) error { - // TODO need to consider how to handle that both contains public/private namespace - for _, resource := range resources { - name := resource.Metadata.Repository.Name - if err := adapter.PrepareForPush(resource); err != nil { - return fmt.Errorf("failed to do the prepare work for pushing/uploading %s: %v", name, err) - } - log.Debugf("the prepare work for pushing/uploading %s completed", name) + if err := adapter.PrepareForPush(resources); err != nil { + return fmt.Errorf("failed to do the prepare work for pushing/uploading resources: %v", err) } + log.Debug("the prepare work for pushing/uploading resources completed") return nil } diff --git a/src/replication/operation/flow/stage_test.go b/src/replication/operation/flow/stage_test.go index cfd710497..d448d1524 100644 --- a/src/replication/operation/flow/stage_test.go +++ b/src/replication/operation/flow/stage_test.go @@ -46,7 +46,7 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) { }, nil } -func (f *fakedAdapter) PrepareForPush(*model.Resource) error { +func (f *fakedAdapter) PrepareForPush([]*model.Resource) error { return nil } func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {