Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-20 15:48:26 +01:00
Code Issues Projects Releases Wiki Activity

Update the PrepareForPush method

Browse Source 
Update the PrepareForPush method to avoid the useless API callings to create namespaces

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2019-04-16 16:40:31 +08:00
parent 0f488e6011
commit 613dfee5ee
10 changed files with 155 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/replication/adapter/adapter.go
View File

@ -30,9 +30,9 @@ type Factory func(*model.Registry) (Adapter, error)
type Adapter interface {
// Info return the information of this adapter
Info() (*model.RegistryInfo, error)
// PrepareForPush does the prepare work that needed for pushing/uploading the resource
// PrepareForPush does the prepare work that needed for pushing/uploading the resources
// eg: create the namespace or repository
PrepareForPush(*model.Resource) error
PrepareForPush([]*model.Resource) error
// HealthCheck checks health status of registry
HealthCheck() (model.HealthStatus, error)
}

56
src/replication/adapter/dockerhub/adapter.go
View File

@ -79,34 +79,40 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
// PrepareForPush does the prepare work that needed for pushing/uploading the resource
// eg: create the namespace or repository
func (a *adapter) PrepareForPush(resource *model.Resource) error {
if resource == nil {
return errors.New("the resource cannot be null")
}
if resource.Metadata == nil {
return errors.New("the metadata of resource cannot be null")
}
if resource.Metadata.Repository == nil {
return errors.New("the namespace of resource cannot be null")
}
if len(resource.Metadata.Repository.Name) == 0 {
return errors.New("the name of the namespace cannot be null")
}
namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name)
// Docker Hub doesn't support the repository contains no "/"
// just skip here and the following task will fail
if len(namespace) == 0 {
log.Debug("the namespace is empty, skip")
return nil
func (a *adapter) PrepareForPush(resources []*model.Resource) error {
namespaces := map[string]struct{}{}
for _, resource := range resources {
if resource == nil {
return errors.New("the resource cannot be null")
}
if resource.Metadata == nil {
return errors.New("the metadata of resource cannot be null")
}
if resource.Metadata.Repository == nil {
return errors.New("the namespace of resource cannot be null")
}
if len(resource.Metadata.Repository.Name) == 0 {
return errors.New("the name of the namespace cannot be null")
}
namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name)
// Docker Hub doesn't support the repository contains no "/"
// just skip here and the following task will fail
if len(namespace) == 0 {
log.Debug("the namespace is empty, skip")
continue
}
namespaces[namespace] = struct{}{}
}
err := a.CreateNamespace(&model.Namespace{
Name: namespace,
})
if err != nil {
return fmt.Errorf("create namespace '%s' in DockerHub error: %v", namespace, err)
for namespace := range namespaces {
err := a.CreateNamespace(&model.Namespace{
Name: namespace,
})
if err != nil {
return fmt.Errorf("create namespace '%s' in DockerHub error: %v", namespace, err)
}
}
return nil
}

66
src/replication/adapter/harbor/adapter.go
View File

@ -136,33 +136,42 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
return info, nil
}
func (a *adapter) PrepareForPush(resource *model.Resource) error {
if resource == nil {
return errors.New("the resource cannot be null")
}
if resource.Metadata == nil {
return errors.New("the metadata of resource cannot be null")
}
if resource.Metadata.Repository == nil {
return errors.New("the repository of resource cannot be null")
}
if len(resource.Metadata.Repository.Name) == 0 {
return errors.New("the name of the repository cannot be null")
}
projectName, _ := util.ParseRepository(resource.Metadata.Repository.Name)
// harbor doesn't support the repository contains no "/"
// just skip here and the following task will fail
if len(projectName) == 0 {
log.Debug("the project name is empty, skip")
return nil
}
project := &struct {
Name string `json:"project_name"`
Metadata map[string]interface{} `json:"metadata"`
}{
Name: projectName,
func (a *adapter) PrepareForPush(resources []*model.Resource) error {
projects := map[string]*project{}
for _, resource := range resources {
if resource == nil {
return errors.New("the resource cannot be null")
}
if resource.Metadata == nil {
return errors.New("the metadata of resource cannot be null")
}
if resource.Metadata.Repository == nil {
return errors.New("the repository of resource cannot be null")
}
if len(resource.Metadata.Repository.Name) == 0 {
return errors.New("the name of the repository cannot be null")
}
projectName, _ := util.ParseRepository(resource.Metadata.Repository.Name)
// harbor doesn't support the repository contains no "/"
// just skip here and the following task will fail
if len(projectName) == 0 {
log.Debug("the project name is empty, skip")
continue
}
// TODO handle the public
projects[projectName] = &project{
Name: projectName,
}
}
for _, project := range projects {
err := a.client.Post(a.coreServiceURL+"/api/projects", project)
if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
log.Debugf("got 409 when trying to create project %s", project.Name)
return nil
}
return err
}
return nil
// TODO
/*
@ -185,13 +194,6 @@ func (a *adapter) PrepareForPush(resource *model.Resource) error {
}
}
*/
err := a.client.Post(a.coreServiceURL+"/api/projects", project)
if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
log.Debugf("got 409 when trying to create project %s", projectName)
return nil
}
return err
}
type project struct {

58
src/replication/adapter/harbor/adapter_test.go
View File

@ -87,31 +87,42 @@ func TestPrepareForPush(t *testing.T) {
adapter, err := newAdapter(registry)
require.Nil(t, err)
// nil resource
err = adapter.PrepareForPush(nil)
err = adapter.PrepareForPush([]*model.Resource{nil})
require.NotNil(t, err)
// nil metadata
err = adapter.PrepareForPush(&model.Resource{})
err = adapter.PrepareForPush([]*model.Resource{
{},
})
require.NotNil(t, err)
// nil repository
err = adapter.PrepareForPush(&model.Resource{
Metadata: &model.ResourceMetadata{},
})
err = adapter.PrepareForPush(
[]*model.Resource{
{
Metadata: &model.ResourceMetadata{},
},
})
require.NotNil(t, err)
// nil repository name
err = adapter.PrepareForPush(&model.Resource{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{},
},
})
err = adapter.PrepareForPush(
[]*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{},
},
},
})
require.NotNil(t, err)
// project doesn't exist
err = adapter.PrepareForPush(&model.Resource{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: "library/hello-world",
err = adapter.PrepareForPush(
[]*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: "library/hello-world",
},
},
},
},
})
})
require.Nil(t, err)
server.Close()
@ -129,12 +140,15 @@ func TestPrepareForPush(t *testing.T) {
}
adapter, err = newAdapter(registry)
require.Nil(t, err)
err = adapter.PrepareForPush(&model.Resource{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: "library/hello-world",
err = adapter.PrepareForPush(
[]*model.Resource{
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: "library/hello-world",
},
},
},
},
})
})
require.Nil(t, err)
}

88
src/replication/adapter/huawei/huawei_adapter.go
View File

@ -120,53 +120,55 @@ func (adapter Adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceM
}
// PrepareForPush prepare for push to Huawei SWR
func (adapter Adapter) PrepareForPush(resource *model.Resource) error {
namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name)
ns, err := adapter.GetNamespace(namespace)
if err != nil {
//
} else {
if ns.Name == namespace {
return nil
func (adapter Adapter) PrepareForPush(resources []*model.Resource) error {
// TODO optimize the logic by merging the same namesapces
for _, resource := range resources {
namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name)
ns, err := adapter.GetNamespace(namespace)
if err != nil {
//
} else {
if ns.Name == namespace {
return nil
}
}
}
url := fmt.Sprintf("%s/dockyard/v2/namespaces", adapter.Registry.URL)
namespacebyte, err := json.Marshal(struct {
Namespace string `json:"namespace"`
}{Namespace: namespace})
if err != nil {
return err
}
r, err := http.NewRequest("POST", url, strings.NewReader(string(namespacebyte)))
if err != nil {
return err
}
r.Header.Add("content-type", "application/json; charset=utf-8")
encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret)))
r.Header.Add("Authorization", "Basic "+encodeAuth)
client := &http.Client{}
if adapter.Registry.Insecure == true {
client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
url := fmt.Sprintf("%s/dockyard/v2/namespaces", adapter.Registry.URL)
namespacebyte, err := json.Marshal(struct {
Namespace string `json:"namespace"`
}{Namespace: namespace})
if err != nil {
return err
}
}
resp, err := client.Do(r)
if err != nil {
return err
}
defer resp.Body.Close()
code := resp.StatusCode
if code >= 300 || code < 200 {
body, _ := ioutil.ReadAll(resp.Body)
return fmt.Errorf("[%d][%s]", code, string(body))
r, err := http.NewRequest("POST", url, strings.NewReader(string(namespacebyte)))
if err != nil {
return err
}
r.Header.Add("content-type", "application/json; charset=utf-8")
encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret)))
r.Header.Add("Authorization", "Basic "+encodeAuth)
client := &http.Client{}
if adapter.Registry.Insecure == true {
client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
}
resp, err := client.Do(r)
if err != nil {
return err
}
defer resp.Body.Close()
code := resp.StatusCode
if code >= 300 || code < 200 {
body, _ := ioutil.ReadAll(resp.Body)
return fmt.Errorf("[%d][%s]", code, string(body))
}
}
return nil
}

2
src/replication/adapter/huawei/huawei_adapter_test.go
View File

@ -49,7 +49,7 @@ func TestAdapter_PrepareForPush(t *testing.T) {
Repository: repository,
}
resource.Metadata = metadata
err := hwAdapter.PrepareForPush(resource)
err := hwAdapter.PrepareForPush([]*model.Resource{resource})
if err != nil {
if strings.HasPrefix(err.Error(), "[401]") {
t.Log("huawei ak/sk is not available", err.Error())

2
src/replication/adapter/native/adapter.go
View File

@ -74,4 +74,4 @@ func (native) Info() (info *model.RegistryInfo, err error) {
}
// PrepareForPush nothing need to do.
func (native) PrepareForPush(*model.Resource) error { return nil }
func (native) PrepareForPush([]*model.Resource) error { return nil }

2
src/replication/operation/controller_test.go
View File

@ -130,7 +130,7 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
}, nil
}
func (f *fakedAdapter) PrepareForPush(*model.Resource) error {
func (f *fakedAdapter) PrepareForPush([]*model.Resource) error {
return nil
}
func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {

10
src/replication/operation/flow/stage.go
View File

@ -209,14 +209,10 @@ func assembleDestinationResources(resources []*model.Resource,
// do the prepare work for pushing/uploading the resources: create the namespace or repository
func prepareForPush(adapter adp.Adapter, resources []*model.Resource) error {
// TODO need to consider how to handle that both contains public/private namespace
for _, resource := range resources {
name := resource.Metadata.Repository.Name
if err := adapter.PrepareForPush(resource); err != nil {
return fmt.Errorf("failed to do the prepare work for pushing/uploading %s: %v", name, err)
}
log.Debugf("the prepare work for pushing/uploading %s completed", name)
if err := adapter.PrepareForPush(resources); err != nil {
return fmt.Errorf("failed to do the prepare work for pushing/uploading resources: %v", err)
}
log.Debug("the prepare work for pushing/uploading resources completed")
return nil
}

2
src/replication/operation/flow/stage_test.go
View File

@ -46,7 +46,7 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
}, nil
}
func (f *fakedAdapter) PrepareForPush(*model.Resource) error {
func (f *fakedAdapter) PrepareForPush([]*model.Resource) error {
return nil
}
func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {