Merge pull request #10389 from lxShaDoWxl/fix/gitlab

Fixed search images in registry gitlab
This commit is contained in:
Wenkai Yin(尹文开) 2020-06-04 14:49:40 +08:00 committed by GitHub
commit b7b1ce457b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 713 additions and 24 deletions

View File

@ -81,31 +81,19 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
var resources []*model.Resource
var projects []*Project
var err error
pattern := ""
nameFilter := ""
tagFilter := ""
for _, filter := range filters {
if filter.Type == model.FilterTypeName {
pattern = filter.Value.(string)
nameFilter = filter.Value.(string)
break
} else if filter.Type == model.FilterTypeTag {
tagFilter = filter.Value.(string)
break
}
}
if len(pattern) > 0 {
substrings := strings.Split(pattern, "/")
projectPattern := substrings[1]
names, ok := util.IsSpecificPathComponent(projectPattern)
if ok {
for _, name := range names {
var projectsByName, err = a.clientGitlabAPI.getProjectsByName(name)
if err != nil {
return nil, err
}
if projectsByName == nil {
continue
}
projects = append(projects, projectsByName...)
}
}
}
projects = a.searchByPattern(nameFilter)
if len(projects) == 0 {
projects, err = a.clientGitlabAPI.getProjects()
if err != nil {
@ -114,8 +102,10 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
}
var pathPatterns []string
if paths, ok := util.IsSpecificPath(pattern); ok {
if paths, ok := util.IsSpecificPath(nameFilter); ok {
pathPatterns = paths
} else {
pathPatterns = append(pathPatterns, nameFilter)
}
for _, project := range projects {
@ -142,9 +132,11 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
}
tags := []string{}
for _, vTag := range vTags {
if !existPatterns(vTag.Path, pathPatterns) {
if len(tagFilter) > 0 {
if ok, _ := util.Match(strings.ToLower(vTag.Name), strings.ToLower(tagFilter)); !ok {
continue
}
}
tags = append(tags, vTag.Name)
}
info := make(map[string]interface{})
@ -167,11 +159,54 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
return resources, nil
}
func (a *adapter) searchByPattern(pattern string) []*Project {
var projects []*Project
var err error
if len(pattern) > 0 {
names, ok := util.IsSpecificPath(pattern)
if ok {
for _, name := range names {
substrings := strings.Split(name, "/")
if len(substrings) != 2 {
continue
}
var projectsByName, err = a.clientGitlabAPI.getProjectsByName(substrings[1])
if err != nil {
return nil
}
if projectsByName == nil {
continue
}
projects = append(projects, projectsByName...)
}
} else {
substrings := strings.Split(pattern, "/")
if len(substrings) != 2 {
return projects
}
projectName := substrings[1]
if projectName == "*" {
return projects
}
projectName = strings.Trim(projectName, "*")
if strings.Contains(projectName, "*") {
return projects
}
projects, err = a.clientGitlabAPI.getProjectsByName(projectName)
if err != nil {
return projects
}
}
}
return projects
}
func existPatterns(path string, patterns []string) bool {
correct := false
if len(patterns) > 0 {
for _, pathPattern := range patterns {
if strings.HasPrefix(strings.ToLower(path), strings.ToLower(pathPattern)) {
if ok, _ := util.Match(strings.ToLower(pathPattern), strings.ToLower(path)); ok {
correct = true
break
}

View File

@ -0,0 +1,125 @@
package gitlab
import (
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"io"
"net/http"
"net/http/httptest"
"os"
"regexp"
"strconv"
"testing"
)
func mustWriteHTTPResponse(t *testing.T, w io.Writer, fixturePath string) {
f, err := os.Open(fixturePath)
if err != nil {
t.Fatalf("error opening fixture file: %v", err)
}
if _, err = io.Copy(w, f); err != nil {
t.Fatalf("error writing response: %v", err)
}
}
func getServer(t *testing.T) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc("/v2/", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Www-Authenticate", "Bearer realm=\"http://"+r.Host+"/jwt/auth\",service=\"container_registry\"")
w.WriteHeader(http.StatusUnauthorized)
})
mux.HandleFunc("/api/v4/projects", func(w http.ResponseWriter, r *http.Request) {
search := r.URL.Query().Get("search")
w.Header().Set("X-Next-Page", "")
switch search {
case "dev-docker":
mustWriteHTTPResponse(t, w, "testdata/projects/dev-docker.json")
break
case "dev-":
mustWriteHTTPResponse(t, w, "testdata/projects/dev-docker.json")
break
case "-docker":
mustWriteHTTPResponse(t, w, "testdata/projects/-docker.json")
break
case "":
mustWriteHTTPResponse(t, w, "testdata/projects/all.json")
break
default:
w.Header().Set("X-Next-Page", "")
w.Write([]byte(`[]`))
break
}
})
for projectID := 1; projectID <= 5; projectID++ {
mux.HandleFunc("/api/v4/projects/"+strconv.Itoa(projectID)+"/registry/repositories", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Next-Page", "")
re := regexp.MustCompile(`projects/(?P<id>\d+)/registry`)
match := re.FindStringSubmatch(r.RequestURI)
mustWriteHTTPResponse(t, w, "testdata/repositories/"+match[1]+".json")
})
for repositoryID := 1; repositoryID <= 5; repositoryID++ {
mux.HandleFunc("/api/v4/projects/"+strconv.Itoa(projectID)+"/registry/repositories/"+strconv.Itoa(repositoryID)+"1/tags", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Next-Page", "")
re := regexp.MustCompile(`repositories/(?P<id>\d+)/tags`)
match := re.FindStringSubmatch(r.RequestURI)
mustWriteHTTPResponse(t, w, "testdata/tags/"+match[1]+".json")
})
}
}
server := httptest.NewServer(mux)
return server
}
func getAdapter(t *testing.T) adp.Adapter {
assertions := assert.New(t)
factory, err := adp.GetFactory(model.RegistryTypeGitLab)
assertions.Nil(err)
assertions.NotNil(factory)
server := getServer(t)
adapter := newAdapter(&model.Registry{
Type: model.RegistryTypeGitLab,
URL: server.URL,
Credential: &model.Credential{
AccessKey: "test",
AccessSecret: "test",
},
})
assertions.Nil(err)
assertions.NotNil(adapter)
return adapter
}
func TestFetchImages(t *testing.T) {
assertions := assert.New(t)
ad := getAdapter(t)
adapter := ad.(*adapter)
templates := map[string]int{
"library/dev-docker": 1,
"library/*-docker": 1,
"library/dev-*": 2,
"*/dev-docker": 1,
"library/*": 2,
}
for k, v := range templates {
resources, err := adapter.FetchArtifacts([]*model.Filter{
{
Type: model.FilterTypeName,
Value: k,
},
})
require.Nil(t, err, k, v)
assertions.Len(resources, v, k, v)
}
}

View File

@ -11,7 +11,6 @@ import (
)
func TestProjects(t *testing.T) {
// chart museum enabled
server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/api/v4/projects",

View File

@ -0,0 +1,97 @@
[
{
"id": 3,
"description": "",
"name": "dev-docker",
"name_with_namespace": "Library / dev-docker",
"path": "dev-docker",
"path_with_namespace": "library/dev-docker",
"created_at": "2019-01-17T09:47:07.504Z",
"default_branch": "master",
"tag_list": [],
"avatar_url": null,
"star_count": 0,
"forks_count": 0,
"last_activity_at": "2019-06-09T15:18:10.045Z",
"empty_repo": false,
"archived": false,
"visibility": "private",
"resolve_outdated_diff_discussions": false,
"container_registry_enabled": true,
"issues_enabled": true,
"merge_requests_enabled": true,
"wiki_enabled": true,
"jobs_enabled": true,
"snippets_enabled": true,
"shared_runners_enabled": true,
"lfs_enabled": true,
"creator_id": 1,
"forked_from_project": {},
"import_status": "finished",
"open_issues_count": 0,
"ci_default_git_depth": null,
"public_jobs": true,
"ci_config_path": null,
"shared_with_groups": [],
"only_allow_merge_if_pipeline_succeeds": false,
"request_access_enabled": false,
"only_allow_merge_if_all_discussions_are_resolved": false,
"printing_merge_request_link_enabled": true,
"merge_method": "merge",
"external_authorization_classification_label": "",
"permissions": {
"project_access": null,
"group_access": null
},
"mirror": false
},
{
"id": 4,
"description": "",
"name": "dev-docker-test",
"name_with_namespace": "Library / dev-docker-test",
"path": "dev-docker-test",
"path_with_namespace": "library/dev-docker-test",
"created_at": "2019-01-17T09:47:07.504Z",
"default_branch": "master",
"tag_list": [],
"avatar_url": null,
"star_count": 0,
"forks_count": 0,
"last_activity_at": "2019-06-09T15:18:10.045Z",
"empty_repo": false,
"archived": false,
"visibility": "private",
"resolve_outdated_diff_discussions": false,
"container_registry_enabled": true,
"issues_enabled": true,
"merge_requests_enabled": true,
"wiki_enabled": true,
"jobs_enabled": true,
"snippets_enabled": true,
"shared_runners_enabled": true,
"lfs_enabled": true,
"creator_id": 1,
"forked_from_project": {},
"import_status": "finished",
"open_issues_count": 0,
"ci_default_git_depth": null,
"public_jobs": true,
"ci_config_path": null,
"shared_with_groups": [],
"only_allow_merge_if_pipeline_succeeds": false,
"request_access_enabled": false,
"only_allow_merge_if_all_discussions_are_resolved": false,
"printing_merge_request_link_enabled": true,
"merge_method": "merge",
"external_authorization_classification_label": "",
"permissions": {
"project_access": null,
"group_access": null
},
"mirror": false
}
]

View File

@ -0,0 +1,96 @@
[
{
"id": 1,
"description": "",
"name": "dockers",
"name_with_namespace": "Library / dockers",
"path": "dockers",
"path_with_namespace": "library/dockers",
"created_at": "2019-01-17T09:47:07.504Z",
"default_branch": "master",
"tag_list": [],
"avatar_url": null,
"star_count": 0,
"forks_count": 0,
"last_activity_at": "2019-06-09T15:18:10.045Z",
"empty_repo": false,
"archived": false,
"visibility": "private",
"resolve_outdated_diff_discussions": false,
"container_registry_enabled": true,
"issues_enabled": true,
"merge_requests_enabled": true,
"wiki_enabled": true,
"jobs_enabled": true,
"snippets_enabled": true,
"shared_runners_enabled": true,
"lfs_enabled": true,
"creator_id": 1,
"forked_from_project": {},
"import_status": "finished",
"open_issues_count": 0,
"ci_default_git_depth": null,
"public_jobs": true,
"ci_config_path": null,
"shared_with_groups": [],
"only_allow_merge_if_pipeline_succeeds": false,
"request_access_enabled": false,
"only_allow_merge_if_all_discussions_are_resolved": false,
"printing_merge_request_link_enabled": true,
"merge_method": "merge",
"external_authorization_classification_label": "",
"permissions": {
"project_access": null,
"group_access": null
},
"mirror": false
},
{
"id": 5,
"description": "",
"name": "example-dockers",
"name_with_namespace": "Library / example-dockers",
"path": "example-dockers",
"path_with_namespace": "library/example-dockers",
"created_at": "2019-01-17T09:47:07.504Z",
"default_branch": "master",
"tag_list": [],
"avatar_url": null,
"star_count": 0,
"forks_count": 0,
"last_activity_at": "2019-06-09T15:18:10.045Z",
"empty_repo": false,
"archived": false,
"visibility": "private",
"resolve_outdated_diff_discussions": false,
"container_registry_enabled": true,
"issues_enabled": true,
"merge_requests_enabled": true,
"wiki_enabled": true,
"jobs_enabled": true,
"snippets_enabled": true,
"shared_runners_enabled": true,
"lfs_enabled": true,
"creator_id": 1,
"forked_from_project": {},
"import_status": "finished",
"open_issues_count": 0,
"ci_default_git_depth": null,
"public_jobs": true,
"ci_config_path": null,
"shared_with_groups": [],
"only_allow_merge_if_pipeline_succeeds": false,
"request_access_enabled": false,
"only_allow_merge_if_all_discussions_are_resolved": false,
"printing_merge_request_link_enabled": true,
"merge_method": "merge",
"external_authorization_classification_label": "",
"permissions": {
"project_access": null,
"group_access": null
},
"mirror": false
}
]

View File

@ -0,0 +1,96 @@
[
{
"id": 3,
"description": "",
"name": "dev-docker",
"name_with_namespace": "Library / dev-docker",
"path": "dev-docker",
"path_with_namespace": "library/dev-docker",
"created_at": "2019-01-17T09:47:07.504Z",
"default_branch": "master",
"tag_list": [],
"avatar_url": null,
"star_count": 0,
"forks_count": 0,
"last_activity_at": "2019-06-09T15:18:10.045Z",
"empty_repo": false,
"archived": false,
"visibility": "private",
"resolve_outdated_diff_discussions": false,
"container_registry_enabled": true,
"issues_enabled": true,
"merge_requests_enabled": true,
"wiki_enabled": true,
"jobs_enabled": true,
"snippets_enabled": true,
"shared_runners_enabled": true,
"lfs_enabled": true,
"creator_id": 1,
"forked_from_project": {},
"import_status": "finished",
"open_issues_count": 0,
"ci_default_git_depth": null,
"public_jobs": true,
"ci_config_path": null,
"shared_with_groups": [],
"only_allow_merge_if_pipeline_succeeds": false,
"request_access_enabled": false,
"only_allow_merge_if_all_discussions_are_resolved": false,
"printing_merge_request_link_enabled": true,
"merge_method": "merge",
"external_authorization_classification_label": "",
"permissions": {
"project_access": null,
"group_access": null
},
"mirror": false
},
{
"id": 4,
"description": "",
"name": "dev-docker-test",
"name_with_namespace": "Library / dev-docker-test",
"path": "dev-docker-test",
"path_with_namespace": "library/dev-docker-test",
"created_at": "2019-01-17T09:47:07.504Z",
"default_branch": "master",
"tag_list": [],
"avatar_url": null,
"star_count": 0,
"forks_count": 0,
"last_activity_at": "2019-06-09T15:18:10.045Z",
"empty_repo": false,
"archived": false,
"visibility": "private",
"resolve_outdated_diff_discussions": false,
"container_registry_enabled": true,
"issues_enabled": true,
"merge_requests_enabled": true,
"wiki_enabled": true,
"jobs_enabled": true,
"snippets_enabled": true,
"shared_runners_enabled": true,
"lfs_enabled": true,
"creator_id": 1,
"forked_from_project": {},
"import_status": "finished",
"open_issues_count": 0,
"ci_default_git_depth": null,
"public_jobs": true,
"ci_config_path": null,
"shared_with_groups": [],
"only_allow_merge_if_pipeline_succeeds": false,
"request_access_enabled": false,
"only_allow_merge_if_all_discussions_are_resolved": false,
"printing_merge_request_link_enabled": true,
"merge_method": "merge",
"external_authorization_classification_label": "",
"permissions": {
"project_access": null,
"group_access": null
},
"mirror": false
}
]

View File

@ -0,0 +1,96 @@
[
{
"id": 3,
"description": "",
"name": "dev-docker",
"name_with_namespace": "Library / dev-docker",
"path": "dev-docker",
"path_with_namespace": "library/dev-docker",
"created_at": "2019-01-17T09:47:07.504Z",
"default_branch": "master",
"tag_list": [],
"avatar_url": null,
"star_count": 0,
"forks_count": 0,
"last_activity_at": "2019-06-09T15:18:10.045Z",
"empty_repo": false,
"archived": false,
"visibility": "private",
"resolve_outdated_diff_discussions": false,
"container_registry_enabled": true,
"issues_enabled": true,
"merge_requests_enabled": true,
"wiki_enabled": true,
"jobs_enabled": true,
"snippets_enabled": true,
"shared_runners_enabled": true,
"lfs_enabled": true,
"creator_id": 1,
"forked_from_project": {},
"import_status": "finished",
"open_issues_count": 0,
"ci_default_git_depth": null,
"public_jobs": true,
"ci_config_path": null,
"shared_with_groups": [],
"only_allow_merge_if_pipeline_succeeds": false,
"request_access_enabled": false,
"only_allow_merge_if_all_discussions_are_resolved": false,
"printing_merge_request_link_enabled": true,
"merge_method": "merge",
"external_authorization_classification_label": "",
"permissions": {
"project_access": null,
"group_access": null
},
"mirror": false
},
{
"id": 4,
"description": "",
"name": "dev-docker-test",
"name_with_namespace": "Library / dev-docker-test",
"path": "dev-docker-test",
"path_with_namespace": "library/dev-docker-test",
"created_at": "2019-01-17T09:47:07.504Z",
"default_branch": "master",
"tag_list": [],
"avatar_url": null,
"star_count": 0,
"forks_count": 0,
"last_activity_at": "2019-06-09T15:18:10.045Z",
"empty_repo": false,
"archived": false,
"visibility": "private",
"resolve_outdated_diff_discussions": false,
"container_registry_enabled": true,
"issues_enabled": true,
"merge_requests_enabled": true,
"wiki_enabled": true,
"jobs_enabled": true,
"snippets_enabled": true,
"shared_runners_enabled": true,
"lfs_enabled": true,
"creator_id": 1,
"forked_from_project": {},
"import_status": "finished",
"open_issues_count": 0,
"ci_default_git_depth": null,
"public_jobs": true,
"ci_config_path": null,
"shared_with_groups": [],
"only_allow_merge_if_pipeline_succeeds": false,
"request_access_enabled": false,
"only_allow_merge_if_all_discussions_are_resolved": false,
"printing_merge_request_link_enabled": true,
"merge_method": "merge",
"external_authorization_classification_label": "",
"permissions": {
"project_access": null,
"group_access": null
},
"mirror": false
}
]

View File

@ -0,0 +1,10 @@
[
{
"id": 11,
"name": "",
"path": "library/dockers",
"project_id": 1,
"location": "registry.gitlab.com/library/dockers",
"created_at": "2019-01-17T09:47:07.504Z"
}
]

View File

@ -0,0 +1,10 @@
[
{
"id": 21,
"name": "",
"path": "library/test-docker",
"project_id": 2,
"location": "registry.gitlab.com/library/test-docker",
"created_at": "2019-01-17T09:47:07.504Z"
}
]

View File

@ -0,0 +1,10 @@
[
{
"id": 31,
"name": "",
"path": "library/dev-docker",
"project_id": 3,
"location": "registry.gitlab.com/library/dev-docker",
"created_at": "2019-01-17T09:47:07.504Z"
}
]

View File

@ -0,0 +1,10 @@
[
{
"id": 41,
"name": "",
"path": "library/dev-docker-test",
"project_id": 4,
"location": "registry.gitlab.com/library/dev-docker-test",
"created_at": "2019-01-17T09:47:07.504Z"
}
]

View File

@ -0,0 +1,10 @@
[
{
"id": 51,
"name": "",
"path": "library/example-dockers",
"project_id": 5,
"location": "registry.gitlab.com/library/example-dockers",
"created_at": "2019-01-17T09:47:07.504Z"
}
]

View File

@ -0,0 +1,17 @@
[
{
"name": "harbor",
"path": "library/dockers:harbor",
"location": "registry.gitlab.com/library/dockers:harbor"
},
{
"name": "latest",
"path": "library/dockers:latest",
"location": "registry.gitlab.com/library/dockers:latest"
},
{
"name": "v2",
"path": "library/dockers:v2",
"location": "registry.gitlab.com/library/dockers:v2"
}
]

View File

@ -0,0 +1,17 @@
[
{
"name": "harbor",
"path": "library/test-docker:harbor",
"location": "registry.gitlab.com/library/test-docker:harbor"
},
{
"name": "latest",
"path": "library/test-docker:latest",
"location": "registry.gitlab.com/library/test-docker:latest"
},
{
"name": "v2",
"path": "library/test-docker:v2",
"location": "registry.gitlab.com/library/test-docker:v2"
}
]

View File

@ -0,0 +1,17 @@
[
{
"name": "harbor",
"path": "library/dev-docker:harbor",
"location": "registry.gitlab.com/library/dev-docker:harbor"
},
{
"name": "latest",
"path": "library/dev-docker:latest",
"location": "registry.gitlab.com/library/dev-docker:latest"
},
{
"name": "v2",
"path": "library/dev-docker:v2",
"location": "registry.gitlab.com/library/dev-docker:v2"
}
]

View File

@ -0,0 +1,22 @@
[
{
"name": "harbor",
"path": "library/dev-docker-test:harbor",
"location": "registry.gitlab.com/library/dev-docker-test:harbor"
},
{
"name": "latest",
"path": "library/dev-docker-test:latest",
"location": "registry.gitlab.com/library/dev-docker-test:latest"
},
{
"name": "v2",
"path": "library/dev-docker-test:v2",
"location": "registry.gitlab.com/library/dev-docker-test:v2"
},
{
"name": "v3",
"path": "library/dev-docker-test:v3",
"location": "registry.gitlab.com/library/dev-docker-test:v3"
}
]

View File

@ -0,0 +1,22 @@
[
{
"name": "harbor",
"path": "library/example-dockers:harbor",
"location": "registry.gitlab.com/library/example-dockers:harbor"
},
{
"name": "latest",
"path": "library/example-dockers:latest",
"location": "registry.gitlab.com/library/example-dockers:latest"
},
{
"name": "v2",
"path": "library/example-dockers:v2",
"location": "registry.gitlab.com/library/example-dockers:v2"
},
{
"name": "v3",
"path": "library/example-dockers:v3",
"location": "registry.gitlab.com/library/example-dockers:v3"
}
]