Implement dockerhub adapter

Signed-off-by: cd1989 <chende@caicloud.io>

src/jobservice/job/impl/replication/ng/replication.go

@@ -28,6 +28,8 @@ import (
 	_ "github.com/goharbor/harbor/src/replication/ng/transfer/repository"
 	// register the Harbor adapter
 	_ "github.com/goharbor/harbor/src/replication/ng/adapter/harbor"
+	// register the DockerHub adapter
+	_ "github.com/goharbor/harbor/src/replication/ng/adapter/dockerhub"
 )
 
 // Replication implements the job interface

src/replication/ng/adapter/dockerhub/adapter.go

@@ -0,0 +1,179 @@
+package dockerhub
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/goharbor/harbor/src/common/utils/log"
+	adp "github.com/goharbor/harbor/src/replication/ng/adapter"
+	"github.com/goharbor/harbor/src/replication/ng/model"
+)
+
+func init() {
+	if err := adp.RegisterFactory(model.RegistryTypeDockerHub, func(registry *model.Registry) (adp.Adapter, error) {
+		client, err := NewClient(registry)
+		if err != nil {
+			return nil, err
+		}
+
+		return &adapter{
+			client:   client,
+			registry: registry,
+			DefaultImageRegistry: adp.NewDefaultImageRegistry(&model.Registry{
+				Name:       registry.Name,
+				URL:        registryURL,
+				Credential: registry.Credential,
+				Insecure:   registry.Insecure,
+			}),
+		}, nil
+	}); err != nil {
+		log.Errorf("Register adapter factory for %s error: %v", model.RegistryTypeDockerHub, err)
+		return
+	}
+	log.Infof("Factory for adapter %s registered", model.RegistryTypeDockerHub)
+}
+
+type adapter struct {
+	*adp.DefaultImageRegistry
+	registry *model.Registry
+	client   *Client
+}
+
+// Ensure '*adapter' implements interface 'Adapter'.
+var _ adp.Adapter = (*adapter)(nil)
+
+// Info returns information of the registry
+func (a *adapter) Info() (*model.RegistryInfo, error) {
+	return &model.RegistryInfo{
+		Type: model.RegistryTypeDockerHub,
+		SupportedResourceTypes: []model.ResourceType{
+			model.ResourceTypeRepository,
+		},
+		SupportedTriggers: []model.TriggerType{
+			model.TriggerTypeManual,
+		},
+	}, nil
+}
+
+// HealthCheck checks health status of the registry
+func (a *adapter) HealthCheck() (model.HealthStatus, error) {
+	return model.Healthy, nil
+}
+
+// ListNamespaces lists namespaces from DockerHub with the provided query conditions.
+func (a *adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Namespace, error) {
+	resp, err := a.client.Do(http.MethodGet, listNamespacePath, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode/100 != 2 {
+		log.Errorf("list namespace error: %s", string(body))
+		return nil, fmt.Errorf("%s", string(body))
+	}
+
+	namespaces := NamespacesResp{}
+	err = json.Unmarshal(body, &namespaces)
+	if err != nil {
+		return nil, err
+	}
+	var result []*model.Namespace
+	for _, ns := range namespaces.Namespaces {
+		// If query set, skip the namespace that doesn't match the query.
+		if query != nil && len(query.Name) > 0 && strings.Index(ns, query.Name) != -1 {
+			continue
+		}
+
+		result = append(result, &model.Namespace{
+			Name: ns,
+		})
+	}
+	return result, nil
+}
+
+// CreateNamespace creates a new namespace in DockerHub
+func (a *adapter) CreateNamespace(namespace *model.Namespace) error {
+	ns, err := a.getNamespace(namespace.Name)
+	if err != nil {
+		return fmt.Errorf("check existence of namespace '%s' error: %v", namespace.Name, err)
+	}
+
+	// If the namespace already exist, return succeeded directly.
+	if ns != nil {
+		log.Infof("Namespace %s already exist in DockerHub, skip it.", namespace.Name)
+		return nil
+	}
+
+	req := &NewOrgReq{
+		Name:     namespace.Name,
+		FullName: namespace.GetStringMetadata(metadataKeyFullName, namespace.Name),
+		Company:  namespace.GetStringMetadata(metadataKeyCompany, namespace.Name),
+	}
+	b, err := json.Marshal(req)
+	if err != nil {
+		return err
+	}
+
+	resp, err := a.client.Do(http.MethodPost, createNamespacePath, bytes.NewReader(b))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode/100 != 2 {
+		log.Errorf("create namespace error: %d -- %s", resp.StatusCode, string(body))
+		return fmt.Errorf("%d -- %s", resp.StatusCode, body)
+	}
+
+	return nil
+}
+
+// GetNamespace gets a namespace from DockerHub.
+func (a *adapter) GetNamespace(namespace string) (*model.Namespace, error) {
+	return &model.Namespace{
+		Name: namespace,
+	}, nil
+}
+
+// getNamespace get namespace from DockerHub, if the namespace not found, two nil would be returned.
+func (a *adapter) getNamespace(namespace string) (*model.Namespace, error) {
+	resp, err := a.client.Do(http.MethodGet, getNamespacePath(namespace), nil)
+
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode == http.StatusNotFound {
+		return nil, nil
+	}
+
+	if resp.StatusCode/100 != 2 {
+		log.Errorf("create namespace error: %d -- %s", resp.StatusCode, string(body))
+		return nil, fmt.Errorf("%d -- %s", resp.StatusCode, body)
+	}
+
+	return &model.Namespace{
+		Name: namespace,
+	}, nil
+}

src/replication/ng/adapter/dockerhub/adapter_test.go

@@ -0,0 +1,68 @@
+package dockerhub
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	adp "github.com/goharbor/harbor/src/replication/ng/adapter"
+	"github.com/goharbor/harbor/src/replication/ng/model"
+)
+
+const (
+	testUser     = ""
+	testPassword = ""
+)
+
+func getAdapter(t *testing.T) adp.Adapter {
+	assert := assert.New(t)
+	factory, err := adp.GetFactory(registryTypeDockerHub)
+	assert.Nil(err)
+	assert.NotNil(factory)
+
+	adapter, err := factory(&model.Registry{
+		Type: registryTypeDockerHub,
+		Credential: &model.Credential{
+			AccessKey:    testUser,
+			AccessSecret: testPassword,
+		},
+	})
+	assert.Nil(err)
+	assert.NotNil(adapter)
+
+	return adapter
+}
+
+func TestListNamespaces(t *testing.T) {
+	if testUser == "" {
+		return
+	}
+
+	assert := assert.New(t)
+	adapter := getAdapter(t)
+
+	namespaces, err := adapter.ListNamespaces(nil)
+	assert.Nil(err)
+	for _, ns := range namespaces {
+		fmt.Println(ns.Name)
+	}
+}
+
+func TestCreateNamespace(t *testing.T) {
+	if testUser == "" {
+		return
+	}
+
+	assert := assert.New(t)
+	adapter := getAdapter(t)
+
+	err := adapter.CreateNamespace(&model.Namespace{
+		Name: "harborns",
+		Metadata: map[string]interface{}{
+			metadataKeyFullName: "harbor namespace",
+			metadataKeyCompany:  "harbor",
+		},
+	})
+	assert.Nil(err)
+}

src/replication/ng/adapter/dockerhub/client.go

@@ -0,0 +1,110 @@
+package dockerhub
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/replication/ng/model"
+)
+
+// Client is a client to interact with DockerHub
+type Client struct {
+	client     *http.Client
+	token      string
+	host       string
+	credential LoginCredential
+}
+
+// NewClient creates a new DockerHub client.
+func NewClient(registry *model.Registry) (*Client, error) {
+	client := &Client{
+		host:   registry.URL,
+		client: http.DefaultClient,
+		credential: LoginCredential{
+			User:     registry.Credential.AccessKey,
+			Password: registry.Credential.AccessSecret,
+		},
+	}
+
+	// Login to DockerHub to get access token, default expire date is 30d.
+	err := client.refreshToken()
+	if err != nil {
+		return nil, fmt.Errorf("login to dockerhub error: %v", err)
+	}
+
+	return client, nil
+}
+
+// refreshToken login to DockerHub with user/password, and retrieve access token.
+func (c *Client) refreshToken() error {
+	b, err := json.Marshal(c.credential)
+	if err != nil {
+		return fmt.Errorf("marshal credential error: %v", err)
+	}
+
+	request, err := http.NewRequest(http.MethodPost, baseURL+loginPath, bytes.NewReader(b))
+	if err != nil {
+		return err
+	}
+	request.Header.Set("Content-Type", "application/json")
+
+	resp, err := c.client.Do(request)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode/100 != 2 {
+		return fmt.Errorf("login to dockerhub error: %s", string(body))
+	}
+
+	token := &TokenResp{}
+	err = json.Unmarshal(body, token)
+	if err != nil {
+		return fmt.Errorf("unmarshal token response error: %v", err)
+	}
+
+	c.token = token.Token
+	return nil
+}
+
+// Do performs http request to DockerHub, it will set token automatically.
+func (c *Client) Do(method, path string, body io.Reader) (*http.Response, error) {
+	url := baseURL + path
+	log.Infof("%s %s", method, url)
+	req, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return nil, err
+	}
+	if body != nil || method == http.MethodPost || method == http.MethodPut {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("JWT %s", c.token))
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		log.Errorf("unexpected error: %v", err)
+		return nil, err
+	}
+
+	if resp.StatusCode/100 != 2 {
+		b, err := ioutil.ReadAll(resp.Body)
+		defer resp.Body.Close()
+		if err != nil {
+			return nil, err
+		}
+
+		return nil, fmt.Errorf("unexpected %d error from dockerhub: %s", resp.StatusCode, string(b))
+	}
+	return resp, nil
+}

src/replication/ng/adapter/dockerhub/consts.go

@@ -0,0 +1,18 @@
+package dockerhub
+
+import "fmt"
+
+const (
+	baseURL             = "https://hub.docker.com"
+	registryURL         = "https://registry-1.docker.io"
+	loginPath           = "/v2/users/login/"
+	listNamespacePath   = "/v2/repositories/namespaces"
+	createNamespacePath = "/v2/orgs/"
+
+	metadataKeyCompany  = "company"
+	metadataKeyFullName = "fullName"
+)
+
+func getNamespacePath(namespace string) string {
+	return fmt.Sprintf("/v2/orgs/%s/", namespace)
+}

src/replication/ng/adapter/dockerhub/types.go

@@ -0,0 +1,34 @@
+package dockerhub
+
+// LoginCredential is request to login.
+type LoginCredential struct {
+	User     string `json:"username"`
+	Password string `json:"password"`
+}
+
+// TokenResp is response of login.
+type TokenResp struct {
+	Token string `json:"token"`
+}
+
+// NamespacesResp is namespace list responsed from DockerHub.
+type NamespacesResp struct {
+	// Namespaces is a list of namespaces
+	Namespaces []string `json:"namespaces"`
+}
+
+// NewOrgReq is request to create a new org as namespace.
+type NewOrgReq struct {
+	// Name is name of the namespace
+	Name string `json:"orgname"`
+	// FullName ...
+	FullName string `json:"full_name"`
+	// Company ...
+	Company string `json:"company"`
+	// Location ...
+	Location string `json:"location"`
+	// ProfileUrl ...
+	ProfileURL string `json:"profile_url"`
+	// GravatarEmail ...
+	GravatarEmail string `json:"gravatar_email"`
+}

src/replication/ng/adapter/harbor/adapter.go

@@ -218,7 +218,6 @@ func (a *adapter) PrepareForPush(resource *model.Resource) error {
 					public = false
 					break
 				}
-
 			}
 			project.Metadata = map[string]interface{}{
 				"public": public,

src/replication/ng/model/namespace.go

@@ -22,6 +22,19 @@ type Namespace struct {
 	Metadata map[string]interface{} `json:"metadata"`
 }
 
+// GetStringMetadata get a string value metadata from the namespace, if not found, return the default value.
+func (n *Namespace) GetStringMetadata(key string, defaultValue string) string {
+	if n.Metadata == nil {
+		return defaultValue
+	}
+
+	if v, ok := n.Metadata[key]; ok {
+		return v.(string)
+	}
+
+	return defaultValue
+}
+
 // NamespaceQuery defines the query condition for listing namespaces
 type NamespaceQuery struct {
 	Name string

src/replication/ng/model/registry.go

@@ -23,7 +23,8 @@ import (
 // const definition
 const (
 	// RegistryTypeHarbor indicates registry type harbor
-	RegistryTypeHarbor RegistryType = "harbor"
+	RegistryTypeHarbor    RegistryType = "harbor"
+	RegistryTypeDockerHub RegistryType = "dockerHub"
 
 	FilterStyleTypeText  = "input"
 	FilterStyleTypeRadio = "radio"

src/replication/ng/replication.go

@@ -30,6 +30,8 @@ import (
 	"github.com/goharbor/harbor/src/replication/ng/registry"
 
 	// register the Harbor adapter
+	_ "github.com/goharbor/harbor/src/replication/ng/adapter/dockerhub"
+	// register the DockerHub adapter
 	_ "github.com/goharbor/harbor/src/replication/ng/adapter/harbor"
 )