feat(Registries): added gitlab adapter

Signed-off-by: lxShaDoWxl <lxshadowxkingxl@gmail.com>

src/jobservice/job/impl/replication/replication.go

@@ -44,6 +44,8 @@ import (
 	_ "github.com/goharbor/harbor/src/replication/adapter/aliacr"
 	// register the Helm Hub adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/helmhub"
+	// register the GitLab adapter
+	_ "github.com/goharbor/harbor/src/replication/adapter/gitlab"
 )
 
 // Replication implements the job interface

src/replication/adapter/gitlab/adapter.go

@@ -0,0 +1,218 @@
+package gitlab
+
+import (
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/common/utils/registry/auth"
+	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/adapter/native"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/goharbor/harbor/src/replication/util"
+	"net/http"
+	"strings"
+)
+
+func init() {
+	if err := adp.RegisterFactory(model.RegistryTypeGitLab, func(registry *model.Registry) (adp.Adapter, error) {
+		return newAdapter(registry)
+	}); err != nil {
+		log.Errorf("failed to register factory for %s: %v", model.RegistryTypeGitLab, err)
+		return
+	}
+	log.Infof("the factory for adapter %s registered", model.RegistryTypeGitLab)
+}
+
+type adapter struct {
+	*native.Adapter
+	registry        *model.Registry
+	url             string
+	username        string
+	token           string
+	clientGitlabApi *Client
+}
+
+func newAdapter(registry *model.Registry) (*adapter, error) {
+
+	var credential auth.Credential
+	if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 {
+		credential = auth.NewBasicAuthCredential(
+			registry.Credential.AccessKey,
+			registry.Credential.AccessSecret)
+	}
+	authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
+		Transport: util.GetHTTPTransport(registry.Insecure),
+	}, credential)
+
+	dockerRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(&model.Registry{
+		Name:       registry.Name,
+		URL:        registry.URL, // specify the URL of Gitlab registry service
+		Credential: registry.Credential,
+		Insecure:   registry.Insecure,
+	}, authorizer)
+	if err != nil {
+		return nil, err
+	}
+
+	return &adapter{
+		registry:        registry,
+		url:             registry.URL,
+		clientGitlabApi: NewClient(registry),
+		Adapter:         dockerRegistryAdapter,
+	}, nil
+}
+
+func (a *adapter) Info() (info *model.RegistryInfo, err error) {
+	return &model.RegistryInfo{
+		Type: model.RegistryTypeGitLab,
+		SupportedResourceTypes: []model.ResourceType{
+			model.ResourceTypeImage,
+		},
+		SupportedResourceFilters: []*model.FilterStyle{
+			{
+				Type:  model.FilterTypeName,
+				Style: model.FilterStyleTypeText,
+			},
+			{
+				Type:  model.FilterTypeTag,
+				Style: model.FilterStyleTypeText,
+			},
+		},
+		SupportedTriggers: []model.TriggerType{
+			model.TriggerTypeManual,
+			model.TriggerTypeScheduled,
+		},
+	}, nil
+}
+
+// FetchImages fetches images
+func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
+	var resources []*model.Resource
+	var projects []*Project
+	var err error
+	pattern := ""
+	for _, filter := range filters {
+		if filter.Type == model.FilterTypeName {
+			pattern = filter.Value.(string)
+			break
+		}
+	}
+
+	if len(pattern) > 0 {
+		substrings := strings.Split(pattern, "/")
+		projectPattern := substrings[1]
+		names, ok := util.IsSpecificPathComponent(projectPattern)
+		if ok {
+			for _, name := range names {
+				var projectsByName, err = a.clientGitlabApi.getProjectsByName(name)
+				if err != nil {
+					return nil, err
+				}
+				if projectsByName == nil {
+					continue
+				}
+				projects = append(projects, projectsByName...)
+			}
+		}
+	}
+	if len(projects) == 0 {
+		projects, err = a.clientGitlabApi.getProjects()
+		if err != nil {
+			return nil, err
+		}
+	}
+	var pathPatterns []string
+
+	if paths, ok := util.IsSpecificPath(pattern); ok {
+		pathPatterns = paths
+	}
+
+	for _, project := range projects {
+		if !existPatterns(project.FullPath, pathPatterns) {
+			continue
+		}
+		repositories, err := a.clientGitlabApi.getRepositories(project.ID)
+		if err != nil {
+			return nil, err
+		}
+		if len(repositories) == 0 {
+			continue
+		}
+		for _, repository := range repositories {
+			if !existPatterns(repository.Path, pathPatterns) {
+				continue
+			}
+			vTags, err := a.clientGitlabApi.getTags(project.ID, repository.ID)
+			if err != nil {
+				return nil, err
+			}
+			if len(vTags) == 0 {
+				continue
+			}
+			tags := []string{}
+			for _, vTag := range vTags {
+				if !existPatterns(vTag.Path, pathPatterns) {
+					continue
+				}
+				tags = append(tags, vTag.Name)
+			}
+			info := make(map[string]interface{})
+			info["location"] = repository.Location
+			info["path"] = repository.Path
+
+			resources = append(resources, &model.Resource{
+				Type:     model.ResourceTypeImage,
+				Registry: a.registry,
+				Metadata: &model.ResourceMetadata{
+					Repository: &model.Repository{
+						Name:     strings.ToLower(repository.Path),
+						Metadata: info,
+					},
+					Vtags: tags,
+				},
+			})
+		}
+	}
+	return resources, nil
+}
+
+func existPatterns(path string, patterns []string) bool {
+	correct := false
+	if len(patterns) > 0 {
+		for _, pathPattern := range patterns {
+			if strings.HasPrefix(strings.ToLower(path), strings.ToLower(pathPattern)) {
+				correct = true
+				break
+			}
+		}
+	} else {
+		correct = true
+	}
+	return correct
+}
+
+////TODO maybe remove and add input form to registry host
+//func (a *adapter) PrepareForPush(resources []*model.Resource) error {
+//	//for _, resource := range resources {
+//	//	var location, err = url.Parse(fmt.Sprintf("%v", resource.Metadata.Repository.Metadata["location"]))
+//	//	if err != nil {
+//	//		return err
+//	//	}
+//	//	endpoint := a.Adapter.Registry.Endpoint
+//	//	endpoint.Host = location.Host
+//	//	a.Adapter.Registry.Endpoint = endpoint
+//	//	break
+//	//}
+//
+//	return nil
+//}
+
+//// PullManifest ...
+//func (a *adapter) PullManifest(repository, reference string, accepttedMediaTypes []string) (distribution.Manifest, string, error) {
+//	//var location, err = url.Parse(repository)
+//	//if err != nil {
+//	//	return nil, "", err
+//	//}
+//	//endpoint := a.Adapter.Registry.Endpoint
+//	//endpoint.Host = location.Host
+//	//a.Adapter.Registry.Endpoint = endpoint
+//	return a.Adapter.PullManifest(repository, reference, accepttedMediaTypes)
+//}

src/replication/adapter/gitlab/client.go

@@ -0,0 +1,182 @@
+package gitlab
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/common/utils/registry/auth"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/goharbor/harbor/src/replication/util"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	common_http "github.com/goharbor/harbor/src/common/http"
+	"net/url"
+	"reflect"
+)
+
+const (
+	scheme = "bearer"
+)
+
+// Client is a client to interact with GitLab
+type Client struct {
+	client   *common_http.Client
+	url      string
+	username string
+	token    string
+}
+
+// NewClient creates a new GitLab client.
+func NewClient(registry *model.Registry) *Client {
+
+	realm, _, err := ping(&http.Client{
+		Transport: util.GetHTTPTransport(registry.Insecure),
+	}, registry.URL)
+	if err != nil {
+		return nil
+	}
+	if realm == "" {
+		return nil
+	}
+	location, err := url.Parse(realm)
+	if err != nil {
+		return nil
+	}
+	client := &Client{
+		url:      location.Scheme + "://" + location.Host,
+		username: registry.Credential.AccessKey,
+		token:    registry.Credential.AccessSecret,
+		client: common_http.NewClient(
+			&http.Client{
+				Transport: util.GetHTTPTransport(registry.Insecure),
+			}),
+	}
+	return client
+}
+
+// ping returns the realm, service and error
+func ping(client *http.Client, endpoint string) (string, string, error) {
+	resp, err := client.Get(buildPingURL(endpoint))
+	if err != nil {
+		return "", "", err
+	}
+	defer resp.Body.Close()
+
+	challenges := auth.ParseChallengeFromResponse(resp)
+	for _, challenge := range challenges {
+		if scheme == challenge.Scheme {
+			realm := challenge.Parameters["realm"]
+			service := challenge.Parameters["service"]
+			return realm, service, nil
+		}
+	}
+
+	log.Warningf("Schemas %v are unsupported", challenges)
+	return "", "", nil
+}
+func buildPingURL(endpoint string) string {
+	return fmt.Sprintf("%s/v2/", endpoint)
+}
+func (c *Client) NewRequest(method, url string, body io.Reader) (*http.Request, error) {
+	req, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("PRIVATE-TOKEN", c.token)
+	return req, nil
+}
+
+func (c *Client) getProjects() ([]*Project, error) {
+	var projects []*Project
+	urlApi := fmt.Sprintf("%s/api/v4/projects?membership=1&per_page=50", c.url)
+	if err := c.GetAndIteratePagination(urlApi, &projects); err != nil {
+		return nil, err
+	}
+	return projects, nil
+}
+
+func (c *Client) getProjectsByName(name string) ([]*Project, error) {
+	var projects []*Project
+	urlApi := fmt.Sprintf("%s/api/v4/projects?search=%s&membership=1&per_page=50", c.url, name)
+	if err := c.GetAndIteratePagination(urlApi, &projects); err != nil {
+		return nil, err
+	}
+	return projects, nil
+}
+func (c *Client) getRepositories(projectID int64) ([]*Repository, error) {
+	var repositories []*Repository
+	urlApi := fmt.Sprintf("%s/api/v4/projects/%d/registry/repositories?per_page=50", c.url, projectID)
+	if err := c.GetAndIteratePagination(urlApi, &repositories); err != nil {
+		return nil, err
+	}
+	return repositories, nil
+}
+
+func (c *Client) getTags(projectID int64, repositoryID int64) ([]*Tag, error) {
+	var tags []*Tag
+	urlApi := fmt.Sprintf("%s/api/v4/projects/%d/registry/repositories/%d/tags?per_page=50", c.url, projectID, repositoryID)
+	if err := c.GetAndIteratePagination(urlApi, &tags); err != nil {
+		return nil, err
+	}
+	return tags, nil
+}
+
+// GetAndIteratePagination iterates the pagination header and returns all resources
+// The parameter "v" must be a pointer to a slice
+func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
+	urlApi, err := url.Parse(endpoint)
+	if err != nil {
+		return err
+	}
+
+	rv := reflect.ValueOf(v)
+	if rv.Kind() != reflect.Ptr {
+		return errors.New("v should be a pointer to a slice")
+	}
+	elemType := rv.Elem().Type()
+	if elemType.Kind() != reflect.Slice {
+		return errors.New("v should be a pointer to a slice")
+	}
+
+	resources := reflect.Indirect(reflect.New(elemType))
+	for len(endpoint) > 0 {
+		req, err := c.NewRequest(http.MethodGet, endpoint, nil)
+		if err != nil {
+			return err
+		}
+		resp, err := c.client.Do(req)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		data, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		if resp.StatusCode < 200 || resp.StatusCode > 299 {
+			return &common_http.Error{
+				Code:    resp.StatusCode,
+				Message: string(data),
+			}
+		}
+
+		res := reflect.New(elemType)
+		if err = json.Unmarshal(data, res.Interface()); err != nil {
+			return err
+		}
+		resources = reflect.AppendSlice(resources, reflect.Indirect(res))
+		endpoint = ""
+
+		nextPage := resp.Header.Get("X-Next-Page")
+		if len(nextPage) > 0 {
+			query := urlApi.Query()
+			query.Set("page", nextPage)
+			endpoint = urlApi.Scheme + "://" + urlApi.Host + urlApi.Path + "?" + query.Encode()
+		}
+	}
+	rv.Elem().Set(resources)
+	return nil
+}

src/replication/adapter/gitlab/client_test.go

@@ -0,0 +1,86 @@
+package gitlab
+
+import (
+	common_http "github.com/goharbor/harbor/src/common/http"
+	"github.com/goharbor/harbor/src/common/utils/test"
+	"github.com/goharbor/harbor/src/replication/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"net/http"
+	"testing"
+)
+
+func TestProjects(t *testing.T) {
+	// chart museum enabled
+	server := test.NewServer(&test.RequestHandlerMapping{
+		Method:  http.MethodGet,
+		Pattern: "/api/v4/projects",
+		Handler: func(w http.ResponseWriter, r *http.Request) {
+			data := `[
+
+{
+        "id": 12312344,
+        "description": "",
+        "name": "dockers",
+        "name_with_namespace": "Library / dockers",
+        "path": "dockers",
+        "path_with_namespace": "library/dockers",
+        "created_at": "2019-01-17T09:47:07.504Z",
+        "default_branch": "master",
+        "tag_list": [],
+    
+        "avatar_url": null,
+        "star_count": 0,
+        "forks_count": 0,
+        "last_activity_at": "2019-06-09T15:18:10.045Z",
+        "empty_repo": false,
+        "archived": false,
+        "visibility": "private",
+        "resolve_outdated_diff_discussions": false,
+        "container_registry_enabled": true,
+        "issues_enabled": true,
+        "merge_requests_enabled": true,
+        "wiki_enabled": true,
+        "jobs_enabled": true,
+        "snippets_enabled": true,
+        "shared_runners_enabled": true,
+        "lfs_enabled": true,
+        "creator_id": 123412412,
+        "forked_from_project": {},
+        "import_status": "finished",
+        "open_issues_count": 0,
+        "ci_default_git_depth": null,
+        "public_jobs": true,
+        "ci_config_path": null,
+        "shared_with_groups": [],
+        "only_allow_merge_if_pipeline_succeeds": false,
+        "request_access_enabled": false,
+        "only_allow_merge_if_all_discussions_are_resolved": false,
+        "printing_merge_request_link_enabled": true,
+        "merge_method": "merge",
+        "external_authorization_classification_label": "",
+        "permissions": {
+            "project_access": null,
+            "group_access": null
+        },
+        "mirror": false
+    }
+
+]`
+			w.Header().Set("X-Next-Page", "")
+			w.Write([]byte(data))
+		},
+	})
+	client := &Client{
+		url:      server.URL,
+		username: "test",
+		token:    "test",
+		client: common_http.NewClient(
+			&http.Client{
+				Transport: util.GetHTTPTransport(true),
+			}),
+	}
+	projects, e := client.getProjects()
+	require.Nil(t, e)
+	assert.Equal(t, 1, len(projects))
+}

src/replication/adapter/gitlab/types.go

@@ -0,0 +1,26 @@
+package gitlab
+
+type TokenResp struct {
+	Token string `json:"token"`
+}
+
+type Project struct {
+	ID              int64  `json:"id"`
+	Name            string `json:"name"`
+	FullPath        string `json:"path_with_namespace"`
+	Visibility      string `json:"visibility"`
+	RegistryEnabled bool   `json:"container_registry_enabled"`
+}
+
+type Repository struct {
+	ID       int64  `json:"id"`
+	Name     string `json:"name"`
+	Path     string `json:"path"`
+	Location string `json:"location"`
+}
+
+type Tag struct {
+	Name     string `json:"name"`
+	Path     string `json:"path"`
+	Location string `json:"location"`
+}

src/replication/model/registry.go

@@ -30,6 +30,7 @@ const (
 	RegistryTypeAwsEcr         RegistryType = "aws-ecr"
 	RegistryTypeAzureAcr       RegistryType = "azure-acr"
 	RegistryTypeAliAcr         RegistryType = "ali-acr"
+	RegistryTypeGitLab         RegistryType = "gitlab"
 
 	RegistryTypeHelmHub RegistryType = "helm-hub"
 

src/replication/replication.go

@@ -45,6 +45,8 @@ import (
 	_ "github.com/goharbor/harbor/src/replication/adapter/aliacr"
 	// register the Helm Hub adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/helmhub"
+	// register the GitLab adapter
+	_ "github.com/goharbor/harbor/src/replication/adapter/gitlab"
 )
 
 var (