From 6937731744e5c7265defe5f4f4da122061dd4df8 Mon Sep 17 00:00:00 2001 From: lxShaDoWxl Date: Mon, 9 Sep 2019 20:48:59 +0600 Subject: [PATCH] feat(Registries): added gitlab adapter Signed-off-by: lxShaDoWxl --- .../job/impl/replication/replication.go | 2 + src/replication/adapter/gitlab/adapter.go | 218 ++++++++++++++++++ src/replication/adapter/gitlab/client.go | 182 +++++++++++++++ src/replication/adapter/gitlab/client_test.go | 86 +++++++ src/replication/adapter/gitlab/types.go | 26 +++ src/replication/model/registry.go | 1 + src/replication/replication.go | 2 + 7 files changed, 517 insertions(+) create mode 100644 src/replication/adapter/gitlab/adapter.go create mode 100644 src/replication/adapter/gitlab/client.go create mode 100644 src/replication/adapter/gitlab/client_test.go create mode 100644 src/replication/adapter/gitlab/types.go diff --git a/src/jobservice/job/impl/replication/replication.go b/src/jobservice/job/impl/replication/replication.go index 849c5c0a2..e341313fe 100644 --- a/src/jobservice/job/impl/replication/replication.go +++ b/src/jobservice/job/impl/replication/replication.go @@ -44,6 +44,8 @@ import ( _ "github.com/goharbor/harbor/src/replication/adapter/aliacr" // register the Helm Hub adapter _ "github.com/goharbor/harbor/src/replication/adapter/helmhub" + // register the GitLab adapter + _ "github.com/goharbor/harbor/src/replication/adapter/gitlab" ) // Replication implements the job interface diff --git a/src/replication/adapter/gitlab/adapter.go b/src/replication/adapter/gitlab/adapter.go new file mode 100644 index 000000000..b1ab6d39f --- /dev/null +++ b/src/replication/adapter/gitlab/adapter.go @@ -0,0 +1,218 @@ +package gitlab + +import ( + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/common/utils/registry/auth" + adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/adapter/native" + "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/replication/util" + "net/http" + "strings" +) + +func init() { + if err := adp.RegisterFactory(model.RegistryTypeGitLab, func(registry *model.Registry) (adp.Adapter, error) { + return newAdapter(registry) + }); err != nil { + log.Errorf("failed to register factory for %s: %v", model.RegistryTypeGitLab, err) + return + } + log.Infof("the factory for adapter %s registered", model.RegistryTypeGitLab) +} + +type adapter struct { + *native.Adapter + registry *model.Registry + url string + username string + token string + clientGitlabApi *Client +} + +func newAdapter(registry *model.Registry) (*adapter, error) { + + var credential auth.Credential + if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 { + credential = auth.NewBasicAuthCredential( + registry.Credential.AccessKey, + registry.Credential.AccessSecret) + } + authorizer := auth.NewStandardTokenAuthorizer(&http.Client{ + Transport: util.GetHTTPTransport(registry.Insecure), + }, credential) + + dockerRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(&model.Registry{ + Name: registry.Name, + URL: registry.URL, // specify the URL of Gitlab registry service + Credential: registry.Credential, + Insecure: registry.Insecure, + }, authorizer) + if err != nil { + return nil, err + } + + return &adapter{ + registry: registry, + url: registry.URL, + clientGitlabApi: NewClient(registry), + Adapter: dockerRegistryAdapter, + }, nil +} + +func (a *adapter) Info() (info *model.RegistryInfo, err error) { + return &model.RegistryInfo{ + Type: model.RegistryTypeGitLab, + SupportedResourceTypes: []model.ResourceType{ + model.ResourceTypeImage, + }, + SupportedResourceFilters: []*model.FilterStyle{ + { + Type: model.FilterTypeName, + Style: model.FilterStyleTypeText, + }, + { + Type: model.FilterTypeTag, + Style: model.FilterStyleTypeText, + }, + }, + SupportedTriggers: []model.TriggerType{ + model.TriggerTypeManual, + model.TriggerTypeScheduled, + }, + }, nil +} + +// FetchImages fetches images +func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) { + var resources []*model.Resource + var projects []*Project + var err error + pattern := "" + for _, filter := range filters { + if filter.Type == model.FilterTypeName { + pattern = filter.Value.(string) + break + } + } + + if len(pattern) > 0 { + substrings := strings.Split(pattern, "/") + projectPattern := substrings[1] + names, ok := util.IsSpecificPathComponent(projectPattern) + if ok { + for _, name := range names { + var projectsByName, err = a.clientGitlabApi.getProjectsByName(name) + if err != nil { + return nil, err + } + if projectsByName == nil { + continue + } + projects = append(projects, projectsByName...) + } + } + } + if len(projects) == 0 { + projects, err = a.clientGitlabApi.getProjects() + if err != nil { + return nil, err + } + } + var pathPatterns []string + + if paths, ok := util.IsSpecificPath(pattern); ok { + pathPatterns = paths + } + + for _, project := range projects { + if !existPatterns(project.FullPath, pathPatterns) { + continue + } + repositories, err := a.clientGitlabApi.getRepositories(project.ID) + if err != nil { + return nil, err + } + if len(repositories) == 0 { + continue + } + for _, repository := range repositories { + if !existPatterns(repository.Path, pathPatterns) { + continue + } + vTags, err := a.clientGitlabApi.getTags(project.ID, repository.ID) + if err != nil { + return nil, err + } + if len(vTags) == 0 { + continue + } + tags := []string{} + for _, vTag := range vTags { + if !existPatterns(vTag.Path, pathPatterns) { + continue + } + tags = append(tags, vTag.Name) + } + info := make(map[string]interface{}) + info["location"] = repository.Location + info["path"] = repository.Path + + resources = append(resources, &model.Resource{ + Type: model.ResourceTypeImage, + Registry: a.registry, + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: strings.ToLower(repository.Path), + Metadata: info, + }, + Vtags: tags, + }, + }) + } + } + return resources, nil +} + +func existPatterns(path string, patterns []string) bool { + correct := false + if len(patterns) > 0 { + for _, pathPattern := range patterns { + if strings.HasPrefix(strings.ToLower(path), strings.ToLower(pathPattern)) { + correct = true + break + } + } + } else { + correct = true + } + return correct +} + +////TODO maybe remove and add input form to registry host +//func (a *adapter) PrepareForPush(resources []*model.Resource) error { +// //for _, resource := range resources { +// // var location, err = url.Parse(fmt.Sprintf("%v", resource.Metadata.Repository.Metadata["location"])) +// // if err != nil { +// // return err +// // } +// // endpoint := a.Adapter.Registry.Endpoint +// // endpoint.Host = location.Host +// // a.Adapter.Registry.Endpoint = endpoint +// // break +// //} +// +// return nil +//} + +//// PullManifest ... +//func (a *adapter) PullManifest(repository, reference string, accepttedMediaTypes []string) (distribution.Manifest, string, error) { +// //var location, err = url.Parse(repository) +// //if err != nil { +// // return nil, "", err +// //} +// //endpoint := a.Adapter.Registry.Endpoint +// //endpoint.Host = location.Host +// //a.Adapter.Registry.Endpoint = endpoint +// return a.Adapter.PullManifest(repository, reference, accepttedMediaTypes) +//} diff --git a/src/replication/adapter/gitlab/client.go b/src/replication/adapter/gitlab/client.go new file mode 100644 index 000000000..fd7be2403 --- /dev/null +++ b/src/replication/adapter/gitlab/client.go @@ -0,0 +1,182 @@ +package gitlab + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/common/utils/registry/auth" + "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/replication/util" + "io" + "io/ioutil" + "net/http" + + common_http "github.com/goharbor/harbor/src/common/http" + "net/url" + "reflect" +) + +const ( + scheme = "bearer" +) + +// Client is a client to interact with GitLab +type Client struct { + client *common_http.Client + url string + username string + token string +} + +// NewClient creates a new GitLab client. +func NewClient(registry *model.Registry) *Client { + + realm, _, err := ping(&http.Client{ + Transport: util.GetHTTPTransport(registry.Insecure), + }, registry.URL) + if err != nil { + return nil + } + if realm == "" { + return nil + } + location, err := url.Parse(realm) + if err != nil { + return nil + } + client := &Client{ + url: location.Scheme + "://" + location.Host, + username: registry.Credential.AccessKey, + token: registry.Credential.AccessSecret, + client: common_http.NewClient( + &http.Client{ + Transport: util.GetHTTPTransport(registry.Insecure), + }), + } + return client +} + +// ping returns the realm, service and error +func ping(client *http.Client, endpoint string) (string, string, error) { + resp, err := client.Get(buildPingURL(endpoint)) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + + challenges := auth.ParseChallengeFromResponse(resp) + for _, challenge := range challenges { + if scheme == challenge.Scheme { + realm := challenge.Parameters["realm"] + service := challenge.Parameters["service"] + return realm, service, nil + } + } + + log.Warningf("Schemas %v are unsupported", challenges) + return "", "", nil +} +func buildPingURL(endpoint string) string { + return fmt.Sprintf("%s/v2/", endpoint) +} +func (c *Client) NewRequest(method, url string, body io.Reader) (*http.Request, error) { + req, err := http.NewRequest(method, url, body) + if err != nil { + return nil, err + } + req.Header.Set("PRIVATE-TOKEN", c.token) + return req, nil +} + +func (c *Client) getProjects() ([]*Project, error) { + var projects []*Project + urlApi := fmt.Sprintf("%s/api/v4/projects?membership=1&per_page=50", c.url) + if err := c.GetAndIteratePagination(urlApi, &projects); err != nil { + return nil, err + } + return projects, nil +} + +func (c *Client) getProjectsByName(name string) ([]*Project, error) { + var projects []*Project + urlApi := fmt.Sprintf("%s/api/v4/projects?search=%s&membership=1&per_page=50", c.url, name) + if err := c.GetAndIteratePagination(urlApi, &projects); err != nil { + return nil, err + } + return projects, nil +} +func (c *Client) getRepositories(projectID int64) ([]*Repository, error) { + var repositories []*Repository + urlApi := fmt.Sprintf("%s/api/v4/projects/%d/registry/repositories?per_page=50", c.url, projectID) + if err := c.GetAndIteratePagination(urlApi, &repositories); err != nil { + return nil, err + } + return repositories, nil +} + +func (c *Client) getTags(projectID int64, repositoryID int64) ([]*Tag, error) { + var tags []*Tag + urlApi := fmt.Sprintf("%s/api/v4/projects/%d/registry/repositories/%d/tags?per_page=50", c.url, projectID, repositoryID) + if err := c.GetAndIteratePagination(urlApi, &tags); err != nil { + return nil, err + } + return tags, nil +} + +// GetAndIteratePagination iterates the pagination header and returns all resources +// The parameter "v" must be a pointer to a slice +func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error { + urlApi, err := url.Parse(endpoint) + if err != nil { + return err + } + + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr { + return errors.New("v should be a pointer to a slice") + } + elemType := rv.Elem().Type() + if elemType.Kind() != reflect.Slice { + return errors.New("v should be a pointer to a slice") + } + + resources := reflect.Indirect(reflect.New(elemType)) + for len(endpoint) > 0 { + req, err := c.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return err + } + resp, err := c.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return &common_http.Error{ + Code: resp.StatusCode, + Message: string(data), + } + } + + res := reflect.New(elemType) + if err = json.Unmarshal(data, res.Interface()); err != nil { + return err + } + resources = reflect.AppendSlice(resources, reflect.Indirect(res)) + endpoint = "" + + nextPage := resp.Header.Get("X-Next-Page") + if len(nextPage) > 0 { + query := urlApi.Query() + query.Set("page", nextPage) + endpoint = urlApi.Scheme + "://" + urlApi.Host + urlApi.Path + "?" + query.Encode() + } + } + rv.Elem().Set(resources) + return nil +} diff --git a/src/replication/adapter/gitlab/client_test.go b/src/replication/adapter/gitlab/client_test.go new file mode 100644 index 000000000..20fe9308a --- /dev/null +++ b/src/replication/adapter/gitlab/client_test.go @@ -0,0 +1,86 @@ +package gitlab + +import ( + common_http "github.com/goharbor/harbor/src/common/http" + "github.com/goharbor/harbor/src/common/utils/test" + "github.com/goharbor/harbor/src/replication/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "net/http" + "testing" +) + +func TestProjects(t *testing.T) { + // chart museum enabled + server := test.NewServer(&test.RequestHandlerMapping{ + Method: http.MethodGet, + Pattern: "/api/v4/projects", + Handler: func(w http.ResponseWriter, r *http.Request) { + data := `[ + +{ + "id": 12312344, + "description": "", + "name": "dockers", + "name_with_namespace": "Library / dockers", + "path": "dockers", + "path_with_namespace": "library/dockers", + "created_at": "2019-01-17T09:47:07.504Z", + "default_branch": "master", + "tag_list": [], + + "avatar_url": null, + "star_count": 0, + "forks_count": 0, + "last_activity_at": "2019-06-09T15:18:10.045Z", + "empty_repo": false, + "archived": false, + "visibility": "private", + "resolve_outdated_diff_discussions": false, + "container_registry_enabled": true, + "issues_enabled": true, + "merge_requests_enabled": true, + "wiki_enabled": true, + "jobs_enabled": true, + "snippets_enabled": true, + "shared_runners_enabled": true, + "lfs_enabled": true, + "creator_id": 123412412, + "forked_from_project": {}, + "import_status": "finished", + "open_issues_count": 0, + "ci_default_git_depth": null, + "public_jobs": true, + "ci_config_path": null, + "shared_with_groups": [], + "only_allow_merge_if_pipeline_succeeds": false, + "request_access_enabled": false, + "only_allow_merge_if_all_discussions_are_resolved": false, + "printing_merge_request_link_enabled": true, + "merge_method": "merge", + "external_authorization_classification_label": "", + "permissions": { + "project_access": null, + "group_access": null + }, + "mirror": false + } + +]` + w.Header().Set("X-Next-Page", "") + w.Write([]byte(data)) + }, + }) + client := &Client{ + url: server.URL, + username: "test", + token: "test", + client: common_http.NewClient( + &http.Client{ + Transport: util.GetHTTPTransport(true), + }), + } + projects, e := client.getProjects() + require.Nil(t, e) + assert.Equal(t, 1, len(projects)) +} diff --git a/src/replication/adapter/gitlab/types.go b/src/replication/adapter/gitlab/types.go new file mode 100644 index 000000000..b29f7c01e --- /dev/null +++ b/src/replication/adapter/gitlab/types.go @@ -0,0 +1,26 @@ +package gitlab + +type TokenResp struct { + Token string `json:"token"` +} + +type Project struct { + ID int64 `json:"id"` + Name string `json:"name"` + FullPath string `json:"path_with_namespace"` + Visibility string `json:"visibility"` + RegistryEnabled bool `json:"container_registry_enabled"` +} + +type Repository struct { + ID int64 `json:"id"` + Name string `json:"name"` + Path string `json:"path"` + Location string `json:"location"` +} + +type Tag struct { + Name string `json:"name"` + Path string `json:"path"` + Location string `json:"location"` +} diff --git a/src/replication/model/registry.go b/src/replication/model/registry.go index dfb743cce..fd4a1b8b4 100644 --- a/src/replication/model/registry.go +++ b/src/replication/model/registry.go @@ -30,6 +30,7 @@ const ( RegistryTypeAwsEcr RegistryType = "aws-ecr" RegistryTypeAzureAcr RegistryType = "azure-acr" RegistryTypeAliAcr RegistryType = "ali-acr" + RegistryTypeGitLab RegistryType = "gitlab" RegistryTypeHelmHub RegistryType = "helm-hub" diff --git a/src/replication/replication.go b/src/replication/replication.go index 46b0feb3c..7e3c14640 100644 --- a/src/replication/replication.go +++ b/src/replication/replication.go @@ -45,6 +45,8 @@ import ( _ "github.com/goharbor/harbor/src/replication/adapter/aliacr" // register the Helm Hub adapter _ "github.com/goharbor/harbor/src/replication/adapter/helmhub" + // register the GitLab adapter + _ "github.com/goharbor/harbor/src/replication/adapter/gitlab" ) var (