From 7ee009f1d2d3db3880dcd212641d3c1b30cc4c52 Mon Sep 17 00:00:00 2001 From: Yuan Lei <371304458@qq.com> Date: Thu, 18 Apr 2019 20:46:32 +0800 Subject: [PATCH] add huawei adapter to replication module, and adjust some interface after testing Signed-off-by: Yuan Lei <371304458@qq.com> --- .../job/impl/replication/replication.go | 2 + .../adapter/huawei/huawei_adapter.go | 59 +++--- .../adapter/huawei/huawei_adapter_test.go | 14 +- .../adapter/huawei/image_registry.go | 199 ++++++++++++++++-- .../adapter/huawei/image_registry_test.go | 38 +++- src/replication/model/registry.go | 1 + src/replication/replication.go | 2 + 7 files changed, 267 insertions(+), 48 deletions(-) diff --git a/src/jobservice/job/impl/replication/replication.go b/src/jobservice/job/impl/replication/replication.go index e200a7048..e93dd0920 100644 --- a/src/jobservice/job/impl/replication/replication.go +++ b/src/jobservice/job/impl/replication/replication.go @@ -33,6 +33,8 @@ import ( _ "github.com/goharbor/harbor/src/replication/adapter/dockerhub" // register the Native adapter _ "github.com/goharbor/harbor/src/replication/adapter/native" + // register the Huawei adapter + _ "github.com/goharbor/harbor/src/replication/adapter/huawei" ) // Replication implements the job interface diff --git a/src/replication/adapter/huawei/huawei_adapter.go b/src/replication/adapter/huawei/huawei_adapter.go index 256020bea..7d5c85e8e 100644 --- a/src/replication/adapter/huawei/huawei_adapter.go +++ b/src/replication/adapter/huawei/huawei_adapter.go @@ -10,11 +10,10 @@ import ( "regexp" "strings" - "github.com/goharbor/harbor/src/replication/util" - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/replication/adapter" + adp "github.com/goharbor/harbor/src/replication/adapter" "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/replication/util" ) const ( @@ -22,7 +21,7 @@ const ( ) func init() { - err := adapter.RegisterFactory(huawei, AdapterFactory) + err := adp.RegisterFactory(model.RegistryTypeHuawei, AdapterFactory) if err != nil { log.Errorf("failed to register factory for Huawei: %v", err) return @@ -31,12 +30,13 @@ func init() { } // Adapter is for images replications between harbor and Huawei image repository(SWR) -type Adapter struct { - Registry *model.Registry +type adapter struct { + *adp.DefaultImageRegistry + registry *model.Registry } // Info gets info about Huawei SWR -func (adapter Adapter) Info() (*model.RegistryInfo, error) { +func (a *adapter) Info() (*model.RegistryInfo, error) { registryInfo := model.RegistryInfo{ Type: huawei, Description: "Adapter for SWR -- The image registry of Huawei Cloud", @@ -48,10 +48,10 @@ func (adapter Adapter) Info() (*model.RegistryInfo, error) { } // ListNamespaces lists namespaces from Huawei SWR with the provided query conditions. -func (adapter Adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Namespace, error) { +func (a *adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Namespace, error) { var namespaces []*model.Namespace - urls := fmt.Sprintf("%s/dockyard/v2/visible/namespaces", adapter.Registry.URL) + urls := fmt.Sprintf("%s/dockyard/v2/visible/namespaces", a.registry.URL) r, err := http.NewRequest("GET", urls, nil) if err != nil { @@ -59,11 +59,11 @@ func (adapter Adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Nam } r.Header.Add("content-type", "application/json; charset=utf-8") - encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret))) + encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", a.registry.Credential.AccessKey, a.registry.Credential.AccessSecret))) r.Header.Add("Authorization", "Basic "+encodeAuth) client := &http.Client{} - if adapter.Registry.Insecure == true { + if a.registry.Insecure == true { client = &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -110,7 +110,7 @@ func (adapter Adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Nam } // ConvertResourceMetadata convert resource metadata for Huawei SWR -func (adapter Adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) { +func (a *adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) { metadata := &model.ResourceMetadata{ Repository: resourceMetadata.Repository, Vtags: resourceMetadata.Vtags, @@ -120,11 +120,11 @@ func (adapter Adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceM } // PrepareForPush prepare for push to Huawei SWR -func (adapter Adapter) PrepareForPush(resources []*model.Resource) error { +func (a *adapter) PrepareForPush(resources []*model.Resource) error { // TODO optimize the logic by merging the same namesapces for _, resource := range resources { namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name) - ns, err := adapter.GetNamespace(namespace) + ns, err := a.GetNamespace(namespace) if err != nil { // } else { @@ -133,7 +133,7 @@ func (adapter Adapter) PrepareForPush(resources []*model.Resource) error { } } - url := fmt.Sprintf("%s/dockyard/v2/namespaces", adapter.Registry.URL) + url := fmt.Sprintf("%s/dockyard/v2/namespaces", a.registry.URL) namespacebyte, err := json.Marshal(struct { Namespace string `json:"namespace"` }{Namespace: namespace}) @@ -147,11 +147,11 @@ func (adapter Adapter) PrepareForPush(resources []*model.Resource) error { } r.Header.Add("content-type", "application/json; charset=utf-8") - encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret))) + encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", a.registry.Credential.AccessKey, a.registry.Credential.AccessSecret))) r.Header.Add("Authorization", "Basic "+encodeAuth) client := &http.Client{} - if adapter.Registry.Insecure == true { + if a.registry.Insecure == true { client = &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -174,24 +174,24 @@ func (adapter Adapter) PrepareForPush(resources []*model.Resource) error { } // GetNamespace gets a namespace from Huawei SWR -func (adapter Adapter) GetNamespace(namespaceStr string) (*model.Namespace, error) { +func (a *adapter) GetNamespace(namespaceStr string) (*model.Namespace, error) { var namespace = &model.Namespace{ Name: "", Metadata: make(map[string]interface{}), } - urls := fmt.Sprintf("%s/dockyard/v2/namespaces/%s", adapter.Registry.URL, namespaceStr) + urls := fmt.Sprintf("%s/dockyard/v2/namespaces/%s", a.registry.URL, namespaceStr) r, err := http.NewRequest("GET", urls, nil) if err != nil { return namespace, err } r.Header.Add("content-type", "application/json; charset=utf-8") - encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret))) + encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", a.registry.Credential.AccessKey, a.registry.Credential.AccessSecret))) r.Header.Add("Authorization", "Basic "+encodeAuth) var client *http.Client - if adapter.Registry.Insecure == true { + if a.registry.Insecure == true { client = &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -229,17 +229,20 @@ func (adapter Adapter) GetNamespace(namespaceStr string) (*model.Namespace, erro } // HealthCheck check health for huawei SWR -func (adapter Adapter) HealthCheck() (model.HealthStatus, error) { +func (a *adapter) HealthCheck() (model.HealthStatus, error) { return model.Healthy, nil } // AdapterFactory is the factory for huawei adapter -func AdapterFactory(registry *model.Registry) (adapter.Adapter, error) { - var adapter Adapter - - adapter.Registry = registry - - return adapter, nil +func AdapterFactory(registry *model.Registry) (adp.Adapter, error) { + reg, err := adp.NewDefaultImageRegistry(registry) + if err != nil { + return nil, err + } + return &adapter{ + registry: registry, + DefaultImageRegistry: reg, + }, nil } diff --git a/src/replication/adapter/huawei/huawei_adapter_test.go b/src/replication/adapter/huawei/huawei_adapter_test.go index db6eae94a..9c25c8a30 100644 --- a/src/replication/adapter/huawei/huawei_adapter_test.go +++ b/src/replication/adapter/huawei/huawei_adapter_test.go @@ -5,11 +5,11 @@ import ( "strings" "testing" - "github.com/goharbor/harbor/src/replication/adapter" + adp "github.com/goharbor/harbor/src/replication/adapter" "github.com/goharbor/harbor/src/replication/model" ) -var hwAdapter adapter.Adapter +var hwAdapter adp.Adapter func init() { var err error @@ -17,7 +17,7 @@ func init() { ID: 1, Name: "Huawei", Description: "Adapter for SWR -- The image registry of Huawei Cloud", - Type: "huawei", + Type: model.RegistryTypeHuawei, URL: "https://swr.cn-north-1.myhuaweicloud.com", Credential: &model.Credential{AccessKey: "cn-north-1@AQR6NF5G2MQ1V7U4FCD", AccessSecret: "2f7ec95070592fd4838a3aa4fd09338c047fd1cd654b3422197318f97281cd9"}, Insecure: false, @@ -60,3 +60,11 @@ func TestAdapter_PrepareForPush(t *testing.T) { t.Log("success prepare for push") } } + +func TestAdapter_HealthCheck(t *testing.T) { + health, err := hwAdapter.HealthCheck() + if err != nil { + t.Error(err) + } + t.Log(health) +} diff --git a/src/replication/adapter/huawei/image_registry.go b/src/replication/adapter/huawei/image_registry.go index f37788ccd..fd7e637b5 100644 --- a/src/replication/adapter/huawei/image_registry.go +++ b/src/replication/adapter/huawei/image_registry.go @@ -13,11 +13,11 @@ import ( ) // FetchImages gets resources from Huawei SWR -func (adapter *Adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) { +func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) { resources := []*model.Resource{} - urls := fmt.Sprintf("%s/dockyard/v2/repositories?filter=center::self", adapter.Registry.URL) + urls := fmt.Sprintf("%s/dockyard/v2/repositories?filter=center::self", a.registry.URL) r, err := http.NewRequest("GET", urls, nil) if err != nil { @@ -25,11 +25,11 @@ func (adapter *Adapter) FetchImages(namespaces []string, filters []*model.Filter } r.Header.Add("content-type", "application/json; charset=utf-8") - encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret))) + encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", a.registry.Credential.AccessKey, a.registry.Credential.AccessSecret))) r.Header.Add("Authorization", "Basic "+encodeAuth) client := &http.Client{} - if adapter.Registry.Insecure == true { + if a.registry.Insecure == true { client = &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -57,18 +57,102 @@ func (adapter *Adapter) FetchImages(namespaces []string, filters []*model.Filter return resources, err } for _, repo := range repos { - for _, namespace := range namespaces { - if repo.NamespaceName == namespace { - resource := parseRepoQueryResultToResource(repo) - resource.Registry = adapter.Registry - resources = append(resources, resource) - } - } + resource := parseRepoQueryResultToResource(repo) + resource.Registry = a.registry + resources = append(resources, resource) } return resources, nil } +// ManifestExist check the manifest of Huawei SWR +func (a *adapter) ManifestExist(repository, reference string) (exist bool, digest string, err error) { + token, err := getJwtToken(a, repository) + if err != nil { + return exist, digest, err + } + + urls := fmt.Sprintf("%s/v2/%s/manifests/%s", a.registry.URL, repository, reference) + + r, err := http.NewRequest("GET", urls, nil) + if err != nil { + return exist, digest, err + } + + r.Header.Add("content-type", "application/json; charset=utf-8") + r.Header.Add("Authorization", "Bearer "+token.Token) + + client := &http.Client{} + if a.registry.Insecure == true { + client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + } + resp, err := client.Do(r) + if err != nil { + return exist, digest, err + } + + defer resp.Body.Close() + code := resp.StatusCode + if code >= 300 || code < 200 { + body, _ := ioutil.ReadAll(resp.Body) + return exist, digest, fmt.Errorf("[%d][%s]", code, string(body)) + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return exist, digest, err + } + exist = true + manifest := hwManifest{} + err = json.Unmarshal(body, &manifest) + if err != nil { + return exist, digest, err + } + return exist, manifest.Config.Digest, nil +} + +// DeleteManifest delete the manifest of Huawei SWR +func (a *adapter) DeleteManifest(repository, reference string) error { + token, err := getJwtToken(a, repository) + if err != nil { + return err + } + + urls := fmt.Sprintf("%s/v2/%s/manifests/%s", a.registry.URL, repository, reference) + + r, err := http.NewRequest("DELETE", urls, nil) + if err != nil { + return err + } + r.Header.Add("content-type", "application/json; charset=utf-8") + r.Header.Add("Authorization", "Bearer "+token.Token) + + client := &http.Client{} + if a.registry.Insecure == true { + client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + } + resp, err := client.Do(r) + if err != nil { + return err + } + + defer resp.Body.Close() + code := resp.StatusCode + if code >= 300 || code < 200 { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("[%d][%s]", code, string(body)) + } + + return nil +} + func parseRepoQueryResultToResource(repo hwRepoQueryResult) *model.Resource { var resource model.Resource info := make(map[string]interface{}) @@ -85,7 +169,7 @@ func parseRepoQueryResultToResource(repo hwRepoQueryResult) *model.Resource { info["total_range"] = repo.TotalRange repository := &model.Repository{ - Name: repo.Name, + Name: fmt.Sprintf("%s/%s", repo.NamespaceName, repo.Name), Metadata: info, } resource.ExtendedInfo = info @@ -123,3 +207,94 @@ type hwRepoQueryResult struct { Status bool `json:"status"` TotalRange int64 `json:"total_range"` } + +func getJwtToken(a *adapter, repository string) (token jwtToken, err error) { + urls := fmt.Sprintf("%s/swr/auth/v2/registry/auth?scope=repository:%s:push,pull", a.registry.URL, repository) + + r, err := http.NewRequest("GET", urls, nil) + if err != nil { + return token, err + } + + r.Header.Add("content-type", "application/json; charset=utf-8") + encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", a.registry.Credential.AccessKey, a.registry.Credential.AccessSecret))) + r.Header.Add("Authorization", "Basic "+encodeAuth) + + client := &http.Client{} + if a.registry.Insecure == true { + client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + } + resp, err := client.Do(r) + if err != nil { + return token, err + } + + defer resp.Body.Close() + code := resp.StatusCode + if code >= 300 || code < 200 { + body, _ := ioutil.ReadAll(resp.Body) + return token, fmt.Errorf("[%d][%s]", code, string(body)) + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return token, err + } + err = json.Unmarshal(body, &token) + if err != nil { + return token, err + } + return token, nil +} + +type jwtToken struct { + Token string `json:"token" description:"token return to user"` + ExpiresIn int `json:"expires_in" description:"describes token will expires in how many seconds later"` + IssuedAt time.Time `json:"issued_at" description:"token issued time"` +} + +type hwManifest struct { + // SchemaVersion is the image manifest schema that this image follows + SchemaVersion int `json:"schemaVersion"` + + // MediaType is the media type of this schema. + MediaType string `json:"mediaType,omitempty"` + + // Config references the image configuration as a blob. + Config hwDescriptor `json:"config"` + + // Layers lists descriptors for the layers referenced by the + // configuration. + Layers []hwDescriptor `json:"layers"` + + // summary keeps the summary infos + Summary hwManifestSummary `json:"-"` +} + +type hwDescriptor struct { + // MediaType describe the type of the content. All text based formats are + // encoded as utf-8. + MediaType string `json:"mediaType,omitempty"` + + // Size in bytes of content. + Size int64 `json:"size,omitempty"` + + // Digest uniquely identifies the content. A byte stream can be verified + // against this digest. + Digest string `json:"digest,omitempty"` + + // URLs contains the source URLs of this content. + URLs []string `json:"urls,omitempty"` + + // depandence + Dependence string `json:"dependence,omitempty"` +} + +type hwManifestSummary struct { + Config string + RepoTags []string + Layers []string +} diff --git a/src/replication/adapter/huawei/image_registry_test.go b/src/replication/adapter/huawei/image_registry_test.go index 498096ec6..8eff4b420 100644 --- a/src/replication/adapter/huawei/image_registry_test.go +++ b/src/replication/adapter/huawei/image_registry_test.go @@ -7,24 +7,24 @@ import ( "github.com/goharbor/harbor/src/replication/model" ) -var HWAdapter Adapter +var HWAdapter adapter func init() { hwRegistry := &model.Registry{ ID: 1, Name: "Huawei", Description: "Adapter for SWR -- The image registry of Huawei Cloud", - Type: "huawei", + Type: model.RegistryTypeHuawei, URL: "https://swr.cn-north-1.myhuaweicloud.com", - Credential: &model.Credential{AccessKey: "cn-north-1@AQR6NF5G2MQ1V7U4FCD", AccessSecret: "2f7ec95070592fd4838a3aa4fd09338c047fd1cd654b3422197318f97281cd9"}, + Credential: &model.Credential{AccessKey: "cn-north-1@IJYZLFBKBFN8LOUITAH", AccessSecret: "f31e8e2b948265afdae32e83722a7705fd43e154585ff69e64108247750e5d"}, Insecure: false, Status: "", } - HWAdapter.Registry = hwRegistry + HWAdapter.registry = hwRegistry } func TestAdapter_FetchImages(t *testing.T) { - resources, err := HWAdapter.FetchImages([]string{"swr_namespace2", "sunday0615"}, nil) + resources, err := HWAdapter.FetchImages(nil) if err != nil { if strings.HasPrefix(err.Error(), "[401]") { t.Log("huawei ak/sk is not available", err.Error()) @@ -37,3 +37,31 @@ func TestAdapter_FetchImages(t *testing.T) { } } } + +func TestAdapter_ManifestExist(t *testing.T) { + exist, digest, err := HWAdapter.ManifestExist("", "") + if err != nil { + if strings.HasPrefix(err.Error(), "[401]") { + t.Log("huawei ak/sk is not available", err.Error()) + } else { + t.Error(err) + } + } else { + if exist { + t.Log(digest) + } + } +} + +func TestAdapter_DeleteManifest(t *testing.T) { + err := HWAdapter.DeleteManifest("sundaymango_mango/hello-world", "latest") + if err != nil { + if strings.HasPrefix(err.Error(), "[401]") { + t.Log("huawei ak/sk is not available", err.Error()) + } else { + t.Error(err) + } + } else { + t.Error("the manifest is deleted") + } +} diff --git a/src/replication/model/registry.go b/src/replication/model/registry.go index 0d9e10390..a32d2bfd7 100644 --- a/src/replication/model/registry.go +++ b/src/replication/model/registry.go @@ -25,6 +25,7 @@ const ( // RegistryTypeHarbor indicates registry type harbor RegistryTypeHarbor RegistryType = "harbor" RegistryTypeDockerHub RegistryType = "dockerHub" + RegistryTypeHuawei RegistryType = "Huawei" FilterStyleTypeText = "input" FilterStyleTypeRadio = "radio" diff --git a/src/replication/replication.go b/src/replication/replication.go index ae5ee0314..48418ec9e 100644 --- a/src/replication/replication.go +++ b/src/replication/replication.go @@ -33,6 +33,8 @@ import ( _ "github.com/goharbor/harbor/src/replication/adapter/dockerhub" // register the Native adapter _ "github.com/goharbor/harbor/src/replication/adapter/native" + // register the huawei adapter + _ "github.com/goharbor/harbor/src/replication/adapter/huawei" ) var (