Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-20 09:15:19 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #7450 from yuanshuhan/replication_ng

Browse Source 
add huawei adapter to replication module, and adjust some interface a…
This commit is contained in:
Steven Zou 2019-04-19 12:52:21 +08:00 committed by GitHub
commit d2c4f19a4c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 267 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/jobservice/job/impl/replication/replication.go
View File

@ -33,6 +33,8 @@ import (
_ "github.com/goharbor/harbor/src/replication/adapter/dockerhub"
// register the Native adapter
_ "github.com/goharbor/harbor/src/replication/adapter/native"
// register the Huawei adapter
_ "github.com/goharbor/harbor/src/replication/adapter/huawei"
)
// Replication implements the job interface

59
src/replication/adapter/huawei/huawei_adapter.go
View File

@ -10,11 +10,10 @@ import (
"regexp"
"strings"
"github.com/goharbor/harbor/src/replication/util"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication/adapter"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
const (
@ -22,7 +21,7 @@ const (
)
func init() {
err := adapter.RegisterFactory(huawei, AdapterFactory)
err := adp.RegisterFactory(model.RegistryTypeHuawei, AdapterFactory)
if err != nil {
log.Errorf("failed to register factory for Huawei: %v", err)
return
@ -31,12 +30,13 @@ func init() {
}
// Adapter is for images replications between harbor and Huawei image repository(SWR)
type Adapter struct {
Registry *model.Registry
type adapter struct {
*adp.DefaultImageRegistry
registry *model.Registry
}
// Info gets info about Huawei SWR
func (adapter Adapter) Info() (*model.RegistryInfo, error) {
func (a *adapter) Info() (*model.RegistryInfo, error) {
registryInfo := model.RegistryInfo{
Type: huawei,
Description: "Adapter for SWR -- The image registry of Huawei Cloud",
@ -48,10 +48,10 @@ func (adapter Adapter) Info() (*model.RegistryInfo, error) {
}
// ListNamespaces lists namespaces from Huawei SWR with the provided query conditions.
func (adapter Adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Namespace, error) {
func (a *adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Namespace, error) {
var namespaces []*model.Namespace
urls := fmt.Sprintf("%s/dockyard/v2/visible/namespaces", adapter.Registry.URL)
urls := fmt.Sprintf("%s/dockyard/v2/visible/namespaces", a.registry.URL)
r, err := http.NewRequest("GET", urls, nil)
if err != nil {
@ -59,11 +59,11 @@ func (adapter Adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Nam
}
r.Header.Add("content-type", "application/json; charset=utf-8")
encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret)))
encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", a.registry.Credential.AccessKey, a.registry.Credential.AccessSecret)))
r.Header.Add("Authorization", "Basic "+encodeAuth)
client := &http.Client{}
if adapter.Registry.Insecure == true {
if a.registry.Insecure == true {
client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
@ -110,7 +110,7 @@ func (adapter Adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Nam
}
// ConvertResourceMetadata convert resource metadata for Huawei SWR
func (adapter Adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) {
func (a *adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) {
metadata := &model.ResourceMetadata{
Repository: resourceMetadata.Repository,
Vtags: resourceMetadata.Vtags,
@ -120,11 +120,11 @@ func (adapter Adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceM
}
// PrepareForPush prepare for push to Huawei SWR
func (adapter Adapter) PrepareForPush(resources []*model.Resource) error {
func (a *adapter) PrepareForPush(resources []*model.Resource) error {
// TODO optimize the logic by merging the same namesapces
for _, resource := range resources {
namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name)
ns, err := adapter.GetNamespace(namespace)
ns, err := a.GetNamespace(namespace)
if err != nil {
//
} else {
@ -133,7 +133,7 @@ func (adapter Adapter) PrepareForPush(resources []*model.Resource) error {
}
}
url := fmt.Sprintf("%s/dockyard/v2/namespaces", adapter.Registry.URL)
url := fmt.Sprintf("%s/dockyard/v2/namespaces", a.registry.URL)
namespacebyte, err := json.Marshal(struct {
Namespace string `json:"namespace"`
}{Namespace: namespace})
@ -147,11 +147,11 @@ func (adapter Adapter) PrepareForPush(resources []*model.Resource) error {
}
r.Header.Add("content-type", "application/json; charset=utf-8")
encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret)))
encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", a.registry.Credential.AccessKey, a.registry.Credential.AccessSecret)))
r.Header.Add("Authorization", "Basic "+encodeAuth)
client := &http.Client{}
if adapter.Registry.Insecure == true {
if a.registry.Insecure == true {
client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
@ -174,24 +174,24 @@ func (adapter Adapter) PrepareForPush(resources []*model.Resource) error {
}
// GetNamespace gets a namespace from Huawei SWR
func (adapter Adapter) GetNamespace(namespaceStr string) (*model.Namespace, error) {
func (a *adapter) GetNamespace(namespaceStr string) (*model.Namespace, error) {
var namespace = &model.Namespace{
Name: "",
Metadata: make(map[string]interface{}),
}
urls := fmt.Sprintf("%s/dockyard/v2/namespaces/%s", adapter.Registry.URL, namespaceStr)
urls := fmt.Sprintf("%s/dockyard/v2/namespaces/%s", a.registry.URL, namespaceStr)
r, err := http.NewRequest("GET", urls, nil)
if err != nil {
return namespace, err
}
r.Header.Add("content-type", "application/json; charset=utf-8")
encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret)))
encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", a.registry.Credential.AccessKey, a.registry.Credential.AccessSecret)))
r.Header.Add("Authorization", "Basic "+encodeAuth)
var client *http.Client
if adapter.Registry.Insecure == true {
if a.registry.Insecure == true {
client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
@ -229,17 +229,20 @@ func (adapter Adapter) GetNamespace(namespaceStr string) (*model.Namespace, erro
}
// HealthCheck check health for huawei SWR
func (adapter Adapter) HealthCheck() (model.HealthStatus, error) {
func (a *adapter) HealthCheck() (model.HealthStatus, error) {
return model.Healthy, nil
}
// AdapterFactory is the factory for huawei adapter
func AdapterFactory(registry *model.Registry) (adapter.Adapter, error) {
var adapter Adapter
adapter.Registry = registry
return adapter, nil
func AdapterFactory(registry *model.Registry) (adp.Adapter, error) {
reg, err := adp.NewDefaultImageRegistry(registry)
if err != nil {
return nil, err
}
return &adapter{
registry: registry,
DefaultImageRegistry: reg,
}, nil
}

14
src/replication/adapter/huawei/huawei_adapter_test.go
View File

@ -5,11 +5,11 @@ import (
"strings"
"testing"
"github.com/goharbor/harbor/src/replication/adapter"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
)
var hwAdapter adapter.Adapter
var hwAdapter adp.Adapter
func init() {
var err error
@ -17,7 +17,7 @@ func init() {
ID: 1,
Name: "Huawei",
Description: "Adapter for SWR -- The image registry of Huawei Cloud",
Type: "huawei",
Type: model.RegistryTypeHuawei,
URL: "https://swr.cn-north-1.myhuaweicloud.com",
Credential: &model.Credential{AccessKey: "cn-north-1@AQR6NF5G2MQ1V7U4FCD", AccessSecret: "2f7ec95070592fd4838a3aa4fd09338c047fd1cd654b3422197318f97281cd9"},
Insecure: false,
@ -60,3 +60,11 @@ func TestAdapter_PrepareForPush(t *testing.T) {
t.Log("success prepare for push")
}
}
func TestAdapter_HealthCheck(t *testing.T) {
health, err := hwAdapter.HealthCheck()
if err != nil {
t.Error(err)
}
t.Log(health)
}

199
src/replication/adapter/huawei/image_registry.go
View File

@ -13,11 +13,11 @@ import (
)
// FetchImages gets resources from Huawei SWR
func (adapter *Adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) {
func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
resources := []*model.Resource{}
urls := fmt.Sprintf("%s/dockyard/v2/repositories?filter=center::self", adapter.Registry.URL)
urls := fmt.Sprintf("%s/dockyard/v2/repositories?filter=center::self", a.registry.URL)
r, err := http.NewRequest("GET", urls, nil)
if err != nil {
@ -25,11 +25,11 @@ func (adapter *Adapter) FetchImages(namespaces []string, filters []*model.Filter
}
r.Header.Add("content-type", "application/json; charset=utf-8")
encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", adapter.Registry.Credential.AccessKey, adapter.Registry.Credential.AccessSecret)))
encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", a.registry.Credential.AccessKey, a.registry.Credential.AccessSecret)))
r.Header.Add("Authorization", "Basic "+encodeAuth)
client := &http.Client{}
if adapter.Registry.Insecure == true {
if a.registry.Insecure == true {
client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
@ -57,18 +57,102 @@ func (adapter *Adapter) FetchImages(namespaces []string, filters []*model.Filter
return resources, err
}
for _, repo := range repos {
for _, namespace := range namespaces {
if repo.NamespaceName == namespace {
resource := parseRepoQueryResultToResource(repo)
resource.Registry = adapter.Registry
resources = append(resources, resource)
}
}
resource := parseRepoQueryResultToResource(repo)
resource.Registry = a.registry
resources = append(resources, resource)
}
return resources, nil
}
// ManifestExist check the manifest of Huawei SWR
func (a *adapter) ManifestExist(repository, reference string) (exist bool, digest string, err error) {
token, err := getJwtToken(a, repository)
if err != nil {
return exist, digest, err
}
urls := fmt.Sprintf("%s/v2/%s/manifests/%s", a.registry.URL, repository, reference)
r, err := http.NewRequest("GET", urls, nil)
if err != nil {
return exist, digest, err
}
r.Header.Add("content-type", "application/json; charset=utf-8")
r.Header.Add("Authorization", "Bearer "+token.Token)
client := &http.Client{}
if a.registry.Insecure == true {
client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
}
resp, err := client.Do(r)
if err != nil {
return exist, digest, err
}
defer resp.Body.Close()
code := resp.StatusCode
if code >= 300 || code < 200 {
body, _ := ioutil.ReadAll(resp.Body)
return exist, digest, fmt.Errorf("[%d][%s]", code, string(body))
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return exist, digest, err
}
exist = true
manifest := hwManifest{}
err = json.Unmarshal(body, &manifest)
if err != nil {
return exist, digest, err
}
return exist, manifest.Config.Digest, nil
}
// DeleteManifest delete the manifest of Huawei SWR
func (a *adapter) DeleteManifest(repository, reference string) error {
token, err := getJwtToken(a, repository)
if err != nil {
return err
}
urls := fmt.Sprintf("%s/v2/%s/manifests/%s", a.registry.URL, repository, reference)
r, err := http.NewRequest("DELETE", urls, nil)
if err != nil {
return err
}
r.Header.Add("content-type", "application/json; charset=utf-8")
r.Header.Add("Authorization", "Bearer "+token.Token)
client := &http.Client{}
if a.registry.Insecure == true {
client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
}
resp, err := client.Do(r)
if err != nil {
return err
}
defer resp.Body.Close()
code := resp.StatusCode
if code >= 300 || code < 200 {
body, _ := ioutil.ReadAll(resp.Body)
return fmt.Errorf("[%d][%s]", code, string(body))
}
return nil
}
func parseRepoQueryResultToResource(repo hwRepoQueryResult) *model.Resource {
var resource model.Resource
info := make(map[string]interface{})
@ -85,7 +169,7 @@ func parseRepoQueryResultToResource(repo hwRepoQueryResult) *model.Resource {
info["total_range"] = repo.TotalRange
repository := &model.Repository{
Name: repo.Name,
Name: fmt.Sprintf("%s/%s", repo.NamespaceName, repo.Name),
Metadata: info,
}
resource.ExtendedInfo = info
@ -123,3 +207,94 @@ type hwRepoQueryResult struct {
Status bool `json:"status"`
TotalRange int64 `json:"total_range"`
}
func getJwtToken(a *adapter, repository string) (token jwtToken, err error) {
urls := fmt.Sprintf("%s/swr/auth/v2/registry/auth?scope=repository:%s:push,pull", a.registry.URL, repository)
r, err := http.NewRequest("GET", urls, nil)
if err != nil {
return token, err
}
r.Header.Add("content-type", "application/json; charset=utf-8")
encodeAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", a.registry.Credential.AccessKey, a.registry.Credential.AccessSecret)))
r.Header.Add("Authorization", "Basic "+encodeAuth)
client := &http.Client{}
if a.registry.Insecure == true {
client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
}
resp, err := client.Do(r)
if err != nil {
return token, err
}
defer resp.Body.Close()
code := resp.StatusCode
if code >= 300 || code < 200 {
body, _ := ioutil.ReadAll(resp.Body)
return token, fmt.Errorf("[%d][%s]", code, string(body))
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return token, err
}
err = json.Unmarshal(body, &token)
if err != nil {
return token, err
}
return token, nil
}
type jwtToken struct {
Token string `json:"token" description:"token return to user"`
ExpiresIn int `json:"expires_in" description:"describes token will expires in how many seconds later"`
IssuedAt time.Time `json:"issued_at" description:"token issued time"`
}
type hwManifest struct {
// SchemaVersion is the image manifest schema that this image follows
SchemaVersion int `json:"schemaVersion"`
// MediaType is the media type of this schema.
MediaType string `json:"mediaType,omitempty"`
// Config references the image configuration as a blob.
Config hwDescriptor `json:"config"`
// Layers lists descriptors for the layers referenced by the
// configuration.
Layers []hwDescriptor `json:"layers"`
// summary keeps the summary infos
Summary hwManifestSummary `json:"-"`
}
type hwDescriptor struct {
// MediaType describe the type of the content. All text based formats are
// encoded as utf-8.
MediaType string `json:"mediaType,omitempty"`
// Size in bytes of content.
Size int64 `json:"size,omitempty"`
// Digest uniquely identifies the content. A byte stream can be verified
// against this digest.
Digest string `json:"digest,omitempty"`
// URLs contains the source URLs of this content.
URLs []string `json:"urls,omitempty"`
// depandence
Dependence string `json:"dependence,omitempty"`
}
type hwManifestSummary struct {
Config string
RepoTags []string
Layers []string
}

38
src/replication/adapter/huawei/image_registry_test.go
View File

@ -7,24 +7,24 @@ import (
"github.com/goharbor/harbor/src/replication/model"
)
var HWAdapter Adapter
var HWAdapter adapter
func init() {
hwRegistry := &model.Registry{
ID: 1,
Name: "Huawei",
Description: "Adapter for SWR -- The image registry of Huawei Cloud",
Type: "huawei",
Type: model.RegistryTypeHuawei,
URL: "https://swr.cn-north-1.myhuaweicloud.com",
Credential: &model.Credential{AccessKey: "cn-north-1@AQR6NF5G2MQ1V7U4FCD", AccessSecret: "2f7ec95070592fd4838a3aa4fd09338c047fd1cd654b3422197318f97281cd9"},
Credential: &model.Credential{AccessKey: "cn-north-1@IJYZLFBKBFN8LOUITAH", AccessSecret: "f31e8e2b948265afdae32e83722a7705fd43e154585ff69e64108247750e5d"},
Insecure: false,
Status: "",
}
HWAdapter.Registry = hwRegistry
HWAdapter.registry = hwRegistry
}
func TestAdapter_FetchImages(t *testing.T) {
resources, err := HWAdapter.FetchImages([]string{"swr_namespace2", "sunday0615"}, nil)
resources, err := HWAdapter.FetchImages(nil)
if err != nil {
if strings.HasPrefix(err.Error(), "[401]") {
t.Log("huawei ak/sk is not available", err.Error())
@ -37,3 +37,31 @@ func TestAdapter_FetchImages(t *testing.T) {
}
}
}
func TestAdapter_ManifestExist(t *testing.T) {
exist, digest, err := HWAdapter.ManifestExist("", "")
if err != nil {
if strings.HasPrefix(err.Error(), "[401]") {
t.Log("huawei ak/sk is not available", err.Error())
} else {
t.Error(err)
}
} else {
if exist {
t.Log(digest)
}
}
}
func TestAdapter_DeleteManifest(t *testing.T) {
err := HWAdapter.DeleteManifest("sundaymango_mango/hello-world", "latest")
if err != nil {
if strings.HasPrefix(err.Error(), "[401]") {
t.Log("huawei ak/sk is not available", err.Error())
} else {
t.Error(err)
}
} else {
t.Error("the manifest is deleted")
}
}

1
src/replication/model/registry.go
View File

@ -25,6 +25,7 @@ const (
// RegistryTypeHarbor indicates registry type harbor
RegistryTypeHarbor RegistryType = "harbor"
RegistryTypeDockerHub RegistryType = "dockerHub"
RegistryTypeHuawei RegistryType = "Huawei"
FilterStyleTypeText = "input"
FilterStyleTypeRadio = "radio"

2
src/replication/replication.go
View File

@ -33,6 +33,8 @@ import (
_ "github.com/goharbor/harbor/src/replication/adapter/dockerhub"
// register the Native adapter
_ "github.com/goharbor/harbor/src/replication/adapter/native"
// register the huawei adapter
_ "github.com/goharbor/harbor/src/replication/adapter/huawei"
)
var (