Merge pull request #7163 from yuanshuhan/replication_ng

huawei cloud registry adapter for replication

src/replication/ng/adapter/huawei/huawei_adapter.go

@@ -0,0 +1,269 @@
+package huawei
+
+import (
+	"crypto/tls"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/replication/ng/adapter"
+	"github.com/goharbor/harbor/src/replication/ng/model"
+)
+
+const (
+	huawei model.RegistryType = "Huawei"
+)
+
+func init() {
+	err := adapter.RegisterFactory(huawei, AdapterFactory)
+	if err != nil {
+		log.Errorf("failed to register factory for Huawei: %v", err)
+		return
+	}
+	log.Infof("the factory of Huawei adapter was registered")
+}
+
+// Adapter is for images replications between harbor and Huawei image repository(SWR)
+type Adapter struct {
+	Registry *model.Registry
+}
+
+// Info gets info about Huawei SWR
+func (adapter Adapter) Info() (*model.RegistryInfo, error) {
+	registryInfo := model.RegistryInfo{
+		Type:                     huawei,
+		Description:              "Adapter for SWR -- The image registry of Huawei Cloud",
+		SupportedResourceTypes:   []model.ResourceType{model.ResourceTypeRepository},
+		SupportedResourceFilters: []*model.FilterStyle{},
+		SupportedTriggers:        []model.TriggerType{},
+	}
+	return &registryInfo, nil
+}
+
+// ListNamespaces lists namespaces from Huawei SWR with the provided query conditions.
+func (adapter Adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Namespace, error) {
+	var namespaces []*model.Namespace
+
+	urls := fmt.Sprintf("%s/dockyard/v2/visible/namespaces", adapter.Registry.URL)
+
+	r, err := http.NewRequest("GET", urls, nil)
+	if err != nil {
+		return namespaces, err
+	}
+
+	r.Header.Add("content-type", "application/json; charset=utf-8")
+	encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret)))
+	r.Header.Add("Authorization", "Basic "+encodeAuth)
+
+	client := &http.Client{}
+	if adapter.Registry.Insecure == true {
+		client = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			},
+		}
+	}
+	resp, err := client.Do(r)
+	if err != nil {
+		return namespaces, err
+	}
+
+	defer resp.Body.Close()
+	code := resp.StatusCode
+	if code >= 300 || code < 200 {
+		body, _ := ioutil.ReadAll(resp.Body)
+		return namespaces, fmt.Errorf("[%d][%s]", code, string(body))
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return namespaces, err
+	}
+
+	var namespacesData hwNamespaceList
+	err = json.Unmarshal(body, &namespacesData)
+	if err != nil {
+		return namespaces, err
+	}
+	reg := fmt.Sprintf(".*%s.*", strings.Replace(query.Name, " ", "", -1))
+
+	for _, namespaceData := range namespacesData.Namespace {
+		namespace := model.Namespace{
+			Name:     namespaceData.Name,
+			Metadata: namespaceData.metadata(),
+		}
+		b, err := regexp.MatchString(reg, namespace.Name)
+		if err != nil {
+			return namespaces, nil
+		}
+		if b {
+			namespaces = append(namespaces, &namespace)
+		}
+	}
+	return namespaces, nil
+}
+
+// ConvertResourceMetadata convert resource metadata for Huawei SWR
+func (adapter Adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) {
+	metadata := &model.ResourceMetadata{
+		Namespace:  namespace,
+		Repository: resourceMetadata.Repository,
+		Vtags:      resourceMetadata.Vtags,
+		Labels:     resourceMetadata.Labels,
+	}
+	return metadata, nil
+}
+
+// PrepareForPush prepare for push to Huawei SWR
+func (adapter Adapter) PrepareForPush(resource *model.Resource) error {
+
+	namespace := resource.Metadata.Namespace
+	ns, err := adapter.GetNamespace(namespace.Name)
+	if err != nil {
+		//
+	} else {
+		if ns.Name == namespace.Name {
+			return nil
+		}
+	}
+
+	url := fmt.Sprintf("%s/dockyard/v2/namespaces", adapter.Registry.URL)
+	namespacebyte, err := json.Marshal(struct {
+		Namespace string `json:"namespace"`
+	}{Namespace: namespace.Name})
+	if err != nil {
+		return err
+	}
+
+	r, err := http.NewRequest("POST", url, strings.NewReader(string(namespacebyte)))
+	if err != nil {
+		return err
+	}
+
+	r.Header.Add("content-type", "application/json; charset=utf-8")
+	encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret)))
+	r.Header.Add("Authorization", "Basic "+encodeAuth)
+
+	client := &http.Client{}
+	if adapter.Registry.Insecure == true {
+		client = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			},
+		}
+	}
+	resp, err := client.Do(r)
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+	code := resp.StatusCode
+	if code >= 300 || code < 200 {
+		body, _ := ioutil.ReadAll(resp.Body)
+		return fmt.Errorf("[%d][%s]", code, string(body))
+	}
+	return nil
+}
+
+// GetNamespace gets a namespace from Huawei SWR
+func (adapter Adapter) GetNamespace(namespaceStr string) (*model.Namespace, error) {
+	var namespace = &model.Namespace{
+		Name:     "",
+		Metadata: make(map[string]interface{}),
+	}
+
+	urls := fmt.Sprintf("%s/dockyard/v2/namespaces/%s", adapter.Registry.URL, namespaceStr)
+	r, err := http.NewRequest("GET", urls, nil)
+	if err != nil {
+		return namespace, err
+	}
+
+	r.Header.Add("content-type", "application/json; charset=utf-8")
+	encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret)))
+	r.Header.Add("Authorization", "Basic "+encodeAuth)
+
+	var client *http.Client
+	if adapter.Registry.Insecure == true {
+		client = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			},
+		}
+	} else {
+		client = &http.Client{}
+	}
+	resp, err := client.Do(r)
+	if err != nil {
+		return namespace, err
+	}
+
+	defer resp.Body.Close()
+	code := resp.StatusCode
+	if code >= 300 || code < 200 {
+		body, _ := ioutil.ReadAll(resp.Body)
+		return namespace, fmt.Errorf("[%d][%s]", code, string(body))
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return namespace, err
+	}
+
+	var namespaceData hwNamespace
+	err = json.Unmarshal(body, &namespaceData)
+	if err != nil {
+		return namespace, err
+	}
+
+	namespace.Name = namespaceData.Name
+	namespace.Metadata = namespaceData.metadata()
+
+	return namespace, nil
+}
+
+// HealthCheck check health for huawei SWR
+func (adapter Adapter) HealthCheck() (model.HealthStatus, error) {
+	return model.Healthy, nil
+}
+
+// AdapterFactory is the factory for huawei adapter
+func AdapterFactory(registry *model.Registry) (adapter.Adapter, error) {
+	var adapter Adapter
+
+	adapter.Registry = registry
+
+	return adapter, nil
+
+}
+
+type hwNamespaceList struct {
+	Namespace []hwNamespace `json:"namespaces"`
+}
+
+type hwNamespace struct {
+	ID           int64  `json:"id" orm:"column(id)"`
+	Name         string `json:"name"`
+	CreatorName  string `json:"creator_name,omitempty"`
+	DomainPublic int    `json:"-"`
+	Auth         int    `json:"auth"`
+	DomainName   string `json:"-"`
+	UserCount    int64  `json:"user_count"`
+	ImageCount   int64  `json:"image_count"`
+}
+
+func (ns hwNamespace) metadata() map[string]interface{} {
+	var metadata = make(map[string]interface{})
+	metadata["id"] = ns.ID
+	metadata["creator_name"] = ns.CreatorName
+	metadata["domain_public"] = ns.DomainPublic
+	metadata["auth"] = ns.Auth
+	metadata["domain_name"] = ns.DomainName
+	metadata["user_count"] = ns.UserCount
+	metadata["image_count"] = ns.ImageCount
+
+	return metadata
+}

src/replication/ng/adapter/huawei/huawei_adapter_test.go

@@ -0,0 +1,110 @@
+package huawei
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/goharbor/harbor/src/replication/ng/adapter"
+	"github.com/goharbor/harbor/src/replication/ng/model"
+)
+
+var hwAdapter adapter.Adapter
+
+func init() {
+	var err error
+	hwRegistry := &model.Registry{
+		ID:          1,
+		Name:        "Huawei",
+		Description: "Adapter for SWR -- The image registry of Huawei Cloud",
+		Type:        "huawei",
+		URL:         "https://swr.cn-north-1.myhuaweicloud.com",
+		Credential:  &model.Credential{AccessKey: "cn-north-1@AQR6NF5G2MQ1V7U4FCD", AccessSecret: "2f7ec95070592fd4838a3aa4fd09338c047fd1cd654b3422197318f97281cd9"},
+		Insecure:    false,
+		Status:      "",
+	}
+
+	hwAdapter, err = AdapterFactory(hwRegistry)
+	if err != nil {
+		os.Exit(1)
+	}
+
+}
+
+func TestAdapter_Info(t *testing.T) {
+	info, err := hwAdapter.Info()
+	if err != nil {
+		t.Error(err)
+	}
+	t.Log(info)
+}
+
+func TestAdapter_ListNamespaces(t *testing.T) {
+	namespaces, err := hwAdapter.ListNamespaces(&model.NamespaceQuery{Name: "o"})
+	if err != nil {
+		if strings.HasPrefix(err.Error(), "[401]") {
+			t.Log("huawei ak/sk is not available", err.Error())
+		} else {
+			t.Error(err)
+		}
+	} else {
+		for _, namespace := range namespaces {
+			t.Log(namespace.Name, namespace.Metadata)
+		}
+	}
+}
+
+func TestAdapter_ConvertResourceMetadata(t *testing.T) {
+	metadata := &model.ResourceMetadata{}
+
+	namespace := &model.Namespace{
+		Name:     "domain_repo_new",
+		Metadata: make(map[string]interface{}),
+	}
+
+	metadata, err := hwAdapter.ConvertResourceMetadata(metadata, namespace)
+	if err != nil {
+		if strings.HasPrefix(err.Error(), "[401]") {
+			t.Log("huawei ak/sk is not available", err.Error())
+		} else {
+			t.Error(err)
+		}
+	} else {
+		t.Log("success convert resource metadata")
+		t.Log(metadata)
+	}
+}
+
+func TestAdapter_PrepareForPush(t *testing.T) {
+	namespace := &model.Namespace{
+		Name:     "domain_repo_new",
+		Metadata: make(map[string]interface{}),
+	}
+	resource := &model.Resource{}
+	metadata := &model.ResourceMetadata{Namespace: namespace}
+	resource.Metadata = metadata
+	err := hwAdapter.PrepareForPush(resource)
+	if err != nil {
+		if strings.HasPrefix(err.Error(), "[401]") {
+			t.Log("huawei ak/sk is not available", err.Error())
+		} else {
+			t.Error(err)
+		}
+	} else {
+		t.Log("success prepare for push")
+	}
+}
+
+func TestAdapter_GetNamespace(t *testing.T) {
+	ns, err := hwAdapter.GetNamespace("huaweicloud_namespace_name")
+	if err != nil {
+		if strings.HasPrefix(err.Error(), "[401]") {
+			t.Log("huawei ak/sk is not available", err.Error())
+		} else {
+			t.Error(err)
+		}
+	} else {
+		t.Log(ns)
+	}
+
+}

src/replication/ng/adapter/huawei/image_registry.go

@@ -0,0 +1,130 @@
+package huawei
+
+import (
+	"crypto/tls"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"github.com/goharbor/harbor/src/replication/ng/model"
+)
+
+// FetchImages gets resources from Huawei SWR
+func (adapter *Adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) {
+
+	resources := []*model.Resource{}
+
+	urls := fmt.Sprintf("%s/dockyard/v2/repositories?filter=center::self", adapter.Registry.URL)
+
+	r, err := http.NewRequest("GET", urls, nil)
+	if err != nil {
+		return resources, err
+	}
+
+	r.Header.Add("content-type", "application/json; charset=utf-8")
+	encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret)))
+	r.Header.Add("Authorization", "Basic "+encodeAuth)
+
+	client := &http.Client{}
+	if adapter.Registry.Insecure == true {
+		client = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			},
+		}
+	}
+	resp, err := client.Do(r)
+	if err != nil {
+		return resources, err
+	}
+
+	defer resp.Body.Close()
+	code := resp.StatusCode
+	if code >= 300 || code < 200 {
+		body, _ := ioutil.ReadAll(resp.Body)
+		return resources, fmt.Errorf("[%d][%s]", code, string(body))
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return resources, err
+	}
+	repos := []hwRepoQueryResult{}
+	err = json.Unmarshal(body, &repos)
+	if err != nil {
+		return resources, err
+	}
+	for _, repo := range repos {
+		for _, namespace := range namespaces {
+			if repo.NamespaceName == namespace {
+				resource := parseRepoQueryResultToResource(repo)
+				resource.Registry = adapter.Registry
+				resources = append(resources, resource)
+			}
+		}
+	}
+	return resources, nil
+
+}
+
+func parseRepoQueryResultToResource(repo hwRepoQueryResult) *model.Resource {
+	var resource model.Resource
+	info := make(map[string]interface{})
+	info["category"] = repo.Category
+	info["description"] = repo.Description
+	info["size"] = repo.Size
+	info["is_public"] = repo.IsPublic
+	info["num_images"] = repo.NumImages
+	info["num_download"] = repo.NumDownload
+	info["created_at"] = repo.CreatedAt
+	info["updated_at"] = repo.UpdatedAt
+	info["domain_name"] = repo.DomainName
+	info["status"] = repo.Status
+	info["total_range"] = repo.TotalRange
+
+	namespace := &model.Namespace{
+		Name: repo.NamespaceName,
+	}
+	repository := &model.Repository{
+		Name:     repo.Name,
+		Metadata: info,
+	}
+	resource.ExtendedInfo = info
+	resource.Metadata = &model.ResourceMetadata{
+		Namespace:  namespace,
+		Repository: repository,
+		Vtags:      repo.Tags,
+		Labels:     []string{},
+	}
+	resource.Deleted = false
+	resource.Override = false
+	resource.Type = model.ResourceTypeRepository
+	resource.URI = repo.Path
+
+	return &resource
+}
+
+type hwRepoQueryResult struct {
+	Name        string `json:"name"`
+	Category    string `json:"category"`
+	Description string `json:"description"`
+
+	Size         int64     `json:"size" `
+	IsPublic     bool      `json:"is_public"`
+	NumImages    int64     `json:"num_images"`
+	NumDownload  int64     `json:"num_download"`
+	CreatedAt    time.Time `json:"created_at"`
+	UpdatedAt    time.Time `json:"updated_at"`
+	Logo         string    `json:"logo"`
+	LogoURL      string    `json:"url"`
+	Path         string    `json:"path"`
+	InternalPath string    `json:"internal_path"`
+
+	DomainName    string   `json:"domain_name"`
+	NamespaceName string   `json:"namespace"`
+	Tags          []string `json:"tags"`
+	Status        bool     `json:"status"`
+	TotalRange    int64    `json:"total_range"`
+}

src/replication/ng/adapter/huawei/image_registry_test.go

@@ -0,0 +1,39 @@
+package huawei
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/goharbor/harbor/src/replication/ng/model"
+)
+
+var HWAdapter Adapter
+
+func init() {
+	hwRegistry := &model.Registry{
+		ID:          1,
+		Name:        "Huawei",
+		Description: "Adapter for SWR -- The image registry of Huawei Cloud",
+		Type:        "huawei",
+		URL:         "https://swr.cn-north-1.myhuaweicloud.com",
+		Credential:  &model.Credential{AccessKey: "cn-north-1@AQR6NF5G2MQ1V7U4FCD", AccessSecret: "2f7ec95070592fd4838a3aa4fd09338c047fd1cd654b3422197318f97281cd9"},
+		Insecure:    false,
+		Status:      "",
+	}
+	HWAdapter.Registry = hwRegistry
+}
+
+func TestAdapter_FetchImages(t *testing.T) {
+	resources, err := HWAdapter.FetchImages([]string{"swr_namespace2", "sunday0615"}, nil)
+	if err != nil {
+		if strings.HasPrefix(err.Error(), "[401]") {
+			t.Log("huawei ak/sk is not available", err.Error())
+		} else {
+			t.Error(err)
+		}
+	} else {
+		for _, resource := range resources {
+			t.Log(*resource)
+		}
+	}
+}