Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-09-30 06:18:02 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1518 from ywk253100/170306_search

Browse Source 
Refactor search API to return more info
This commit is contained in:
Daniel Jiang 2017-03-07 13:12:53 +08:00 committed by GitHub
commit c1d9d68232
11 changed files with 149 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
docs/swagger.yaml
View File

@ -1508,26 +1508,12 @@ definitions:
description: Search results of the projects that matched the filter keywords.
type: array
items:
$ref: '#/definitions/SearchProject'
$ref: '#/definitions/Project'
repositories:
description: Search results of the repositories that matched the filter keywords.
type: array
items:
$ref: '#/definitions/SearchRepository'
SearchProject:
type: object
properties:
id:
type: integer
format: int64
description: The ID of project
name:
type: string
description: The name of the project
public:
type: integer
format: int
description: The flag to indicate the publicity of the project (1 is public, 0 is non-public)
SearchRepository:
type: object
properties:
@ -1543,6 +1529,12 @@ definitions:
repository_name:
type: string
description: The name of the repository
pull_count:
type: integer
description: The count how many times the repository is pulled
tags_count:
type: integer
description: The count of tags in the repository
ProjectReq:
type: object
properties:

11
src/common/dao/project.go
View File

@ -168,10 +168,15 @@ func ToggleProjectPublicity(projectID int64, publicity int) error {
// 2. the prject is public or the user is a member of the project
func SearchProjects(userID int) ([]models.Project, error) {
o := GetOrmer()
sql := `select distinct p.project_id, p.name, p.public
sql :=
`select distinct p.project_id, p.name, p.public,
p.owner_id, p.creation_time, p.update_time, pm.role role
from project p
left join project_member pm on p.project_id = pm.project_id
where (pm.user_id = ? or p.public = 1) and p.deleted = 0`
left join project_member pm
on p.project_id = pm.project_id
where (pm.user_id = ? or p.public = 1)
and p.deleted = 0 `
var projects []models.Project

3
src/common/dao/repository.go
View File

@ -50,7 +50,8 @@ func GetRepositoryByName(name string) (*models.RepoRecord, error) {
func GetAllRepositories() ([]models.RepoRecord, error) {
o := GetOrmer()
var repos []models.RepoRecord
_, err := o.QueryTable("repository").All(&repos)
_, err := o.QueryTable("repository").
OrderBy("Name").All(&repos)
return repos, err
}

40
src/ui/api/repository.go
View File

@ -28,7 +28,6 @@ import (
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/common/utils/registry"
"github.com/vmware/harbor/src/ui/service/cache"
svc_utils "github.com/vmware/harbor/src/ui/service/utils"
registry_error "github.com/vmware/harbor/src/common/utils/registry/error"
@ -195,13 +194,6 @@ func (ra *RepositoryAPI) Delete() {
ra.CustomAbort(http.StatusInternalServerError, "")
}
}
go func() {
log.Debug("refreshing catalog cache")
if err := cache.RefreshCatalogCache(); err != nil {
log.Errorf("error occurred while refresh catalog cache: %v", err)
}
}()
}
type tag struct {
@ -234,36 +226,46 @@ func (ra *RepositoryAPI) GetTags() {
}
}
rc, err := ra.initRepositoryClient(repoName)
client, err := ra.initRepositoryClient(repoName)
if err != nil {
log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
tags := []string{}
ts, err := rc.ListTag()
tags, err := listTag(client)
if err != nil {
regErr, ok := err.(*registry_error.Error)
if !ok {
log.Errorf("error occurred while listing tags of %s: %v", repoName, err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
ra.CustomAbort(regErr.StatusCode, regErr.Detail)
}
ra.Data["json"] = tags
ra.ServeJSON()
}
func listTag(client *registry.Repository) ([]string, error) {
tags := []string{}
ts, err := client.ListTag()
if err != nil {
// TODO remove the logic if the bug of registry is fixed
// It's a workaround for a bug of registry: when listing tags of
// a repository which is being pushed, a "NAME_UNKNOWN" error will
// been returned, while the catalog API can list this repository.
if regErr.StatusCode != http.StatusNotFound {
ra.CustomAbort(regErr.StatusCode, regErr.Detail)
if regErr, ok := err.(*registry_error.Error); ok &&
regErr.StatusCode == http.StatusNotFound {
return tags, nil
}
}
tags = append(tags, ts...)
sort.Strings(tags)
ra.Data["json"] = tags
ra.ServeJSON()
return tags, nil
}
// GetManifests handles GET /api/repositories/manifests
@ -382,7 +384,7 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo
return nil, err
}
return cache.NewRepositoryClient(endpoint, !verify, username, repoName,
return NewRepositoryClient(endpoint, !verify, username, repoName,
"repository", repoName, "pull", "push", "*")
}

82
src/ui/api/search.go
View File

@ -20,12 +20,12 @@ import (
"sort"
"strings"
"github.com/vmware/harbor/src/common/api"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/ui/service/cache"
"github.com/vmware/harbor/src/common/utils"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/common/api"
"github.com/vmware/harbor/src/ui/config"
)
// SearchAPI handles requesst to /api/search
@ -34,7 +34,7 @@ type SearchAPI struct {
}
type searchResult struct {
Project []map[string]interface{} `json:"project"`
Project []models.Project `json:"project"`
Repository []map[string]interface{} `json:"repository"`
}
@ -71,58 +71,104 @@ func (s *SearchAPI) Get() {
projectSorter := &models.ProjectSorter{Projects: projects}
sort.Sort(projectSorter)
projectResult := []map[string]interface{}{}
projectResult := []models.Project{}
for _, p := range projects {
match := true
if len(keyword) > 0 && !strings.Contains(p.Name, keyword) {
match = false
}
if match {
entry := make(map[string]interface{})
entry["id"] = p.ProjectID
entry["name"] = p.Name
entry["public"] = p.Public
projectResult = append(projectResult, entry)
if userID != dao.NonExistUserID {
if isSysAdmin {
p.Role = models.PROJECTADMIN
}
if p.Role == models.PROJECTADMIN {
p.Togglable = true
}
}
repositories, err := cache.GetRepoFromCache()
repos, err := dao.GetRepositoryByProjectName(p.Name)
if err != nil {
log.Errorf("failed to list repositories: %v", err)
log.Errorf("failed to get repositories of project %s: %v", p.Name, err)
s.CustomAbort(http.StatusInternalServerError, "")
}
p.RepoCount = len(repos)
projectResult = append(projectResult, p)
}
}
repositoryResult, err := filterRepositories(projects, keyword)
if err != nil {
log.Errorf("failed to filter repositories: %v", err)
s.CustomAbort(http.StatusInternalServerError, "")
}
sort.Strings(repositories)
repositoryResult := filterRepositories(repositories, projects, keyword)
result := &searchResult{Project: projectResult, Repository: repositoryResult}
s.Data["json"] = result
s.ServeJSON()
}
func filterRepositories(repositories []string, projects []models.Project, keyword string) []map[string]interface{} {
func filterRepositories(projects []models.Project, keyword string) (
[]map[string]interface{}, error) {
repositories, err := dao.GetAllRepositories()
if err != nil {
return nil, err
}
i, j := 0, 0
result := []map[string]interface{}{}
for i < len(repositories) && j < len(projects) {
r := repositories[i]
p, _ := utils.ParseRepository(r)
p, _ := utils.ParseRepository(r.Name)
d := strings.Compare(p, projects[j].Name)
if d < 0 {
i++
continue
} else if d == 0 {
i++
if len(keyword) != 0 && !strings.Contains(r, keyword) {
if len(keyword) != 0 && !strings.Contains(r.Name, keyword) {
continue
}
entry := make(map[string]interface{})
entry["repository_name"] = r
entry["repository_name"] = r.Name
entry["project_name"] = projects[j].Name
entry["project_id"] = projects[j].ProjectID
entry["project_public"] = projects[j].Public
entry["pull_count"] = r.PullCount
tags, err := getTags(r.Name)
if err != nil {
return nil, err
}
entry["tags_count"] = len(tags)
result = append(result, entry)
} else {
j++
}
}
return result
return result, nil
}
func getTags(repository string) ([]string, error) {
url, err := config.RegistryURL()
if err != nil {
return nil, err
}
client, err := NewRepositoryClient(url, true,
"admin", repository, "repository", repository, "pull")
if err != nil {
return nil, err
}
tags, err := listTag(client)
if err != nil {
return nil, err
}
return tags, nil
}

4
src/ui/api/search_test.go
View File

@ -22,9 +22,9 @@ func TestSearch(t *testing.T) {
t.Log(err)
} else {
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
assert.Equal(int64(1), result.Projects[0].Id, "Project id should be equal")
assert.Equal(int64(1), result.Projects[0].ProjectID, "Project id should be equal")
assert.Equal("library", result.Projects[0].Name, "Project name should be library")
assert.Equal(int32(1), result.Projects[0].Public, "Project public status should be 1 (true)")
assert.Equal(1, result.Projects[0].Public, "Project public status should be 1 (true)")
}
//--------case 2 : Response Code = 200, sysAdmin and search repo--------//

47
src/ui/api/utils.go
View File

@ -29,9 +29,9 @@ import (
"github.com/vmware/harbor/src/common/utils"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/common/utils/registry"
"github.com/vmware/harbor/src/common/utils/registry/auth"
registry_error "github.com/vmware/harbor/src/common/utils/registry/error"
"github.com/vmware/harbor/src/ui/config"
"github.com/vmware/harbor/src/ui/service/cache"
)
func checkProjectPermission(userID int, projectID int64) bool {
@ -352,7 +352,7 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string
if err != nil {
return needsAdd, needsDel, err
}
client, err := cache.NewRepositoryClient(endpoint, true,
client, err := NewRepositoryClient(endpoint, true,
"admin", repoInR, "repository", repoInR)
if err != nil {
return needsAdd, needsDel, err
@ -377,7 +377,7 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string
if err != nil {
return needsAdd, needsDel, err
}
client, err := cache.NewRepositoryClient(endpoint, true,
client, err := NewRepositoryClient(endpoint, true,
"admin", repoInR, "repository", repoInR)
if err != nil {
return needsAdd, needsDel, err
@ -440,7 +440,7 @@ func initRegistryClient() (r *registry.Registry, err error) {
return nil, err
}
registryClient, err := cache.NewRegistryClient(endpoint, true, "admin",
registryClient, err := NewRegistryClient(endpoint, true, "admin",
"registry", "catalog", "*")
if err != nil {
return nil, err
@ -485,10 +485,6 @@ func getReposByProject(name string, keyword ...string) ([]string, error) {
return repositories, nil
}
func getAllRepos() ([]string, error) {
return cache.GetRepoFromCache()
}
func repositoryExist(name string, client *registry.Repository) (bool, error) {
tags, err := client.ListTag()
if err != nil {
@ -499,3 +495,38 @@ func repositoryExist(name string, client *registry.Repository) (bool, error) {
}
return len(tags) != 0, nil
}
// NewRegistryClient ...
func NewRegistryClient(endpoint string, insecure bool, username, scopeType, scopeName string,
scopeActions ...string) (*registry.Registry, error) {
authorizer := auth.NewUsernameTokenAuthorizer(username, scopeType, scopeName, scopeActions...)
store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
if err != nil {
return nil, err
}
client, err := registry.NewRegistryWithModifiers(endpoint, insecure, store)
if err != nil {
return nil, err
}
return client, nil
}
// NewRepositoryClient ...
func NewRepositoryClient(endpoint string, insecure bool, username, repository, scopeType, scopeName string,
scopeActions ...string) (*registry.Repository, error) {
authorizer := auth.NewUsernameTokenAuthorizer(username, scopeType, scopeName, scopeActions...)
store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
if err != nil {
return nil, err
}
client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store)
if err != nil {
return nil, err
}
return client, nil
}

119
src/ui/service/cache/cache.go vendored
View File

@ -1,119 +0,0 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cache
import (
"time"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/common/utils/registry"
"github.com/vmware/harbor/src/common/utils/registry/auth"
"github.com/astaxie/beego/cache"
)
var (
// Cache is the global cache in system.
Cache cache.Cache
)
const catalogKey string = "catalog"
func init() {
var err error
Cache, err = cache.NewCache("memory", `{"interval":720}`)
if err != nil {
log.Errorf("Failed to initialize cache, error:%v", err)
}
}
// RefreshCatalogCache calls registry's API to get repository list and write it to cache.
func RefreshCatalogCache() error {
log.Debug("refreshing catalog cache...")
repos, err := getAllRepositories()
if err != nil {
return err
}
Cache.Put(catalogKey, repos, 600*time.Second)
return nil
}
// GetRepoFromCache get repository list from cache, it refreshes the cache if it's empty.
func GetRepoFromCache() ([]string, error) {
result := Cache.Get(catalogKey)
if result == nil {
err := RefreshCatalogCache()
if err != nil {
return nil, err
}
cached := Cache.Get(catalogKey)
if cached != nil {
return cached.([]string), nil
}
return nil, nil
}
return result.([]string), nil
}
func getAllRepositories() ([]string, error) {
var repos []string
rs, err := dao.GetAllRepositories()
if err != nil {
return repos, err
}
for _, e := range rs {
repos = append(repos, e.Name)
}
return repos, nil
}
// NewRegistryClient ...
func NewRegistryClient(endpoint string, insecure bool, username, scopeType, scopeName string,
scopeActions ...string) (*registry.Registry, error) {
authorizer := auth.NewUsernameTokenAuthorizer(username, scopeType, scopeName, scopeActions...)
store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
if err != nil {
return nil, err
}
client, err := registry.NewRegistryWithModifiers(endpoint, insecure, store)
if err != nil {
return nil, err
}
return client, nil
}
// NewRepositoryClient ...
func NewRepositoryClient(endpoint string, insecure bool, username, repository, scopeType, scopeName string,
scopeActions ...string) (*registry.Repository, error) {
authorizer := auth.NewUsernameTokenAuthorizer(username, scopeType, scopeName, scopeActions...)
store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
if err != nil {
return nil, err
}
client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store)
if err != nil {
return nil, err
}
return client, nil
}

9
src/ui/service/cache/cache_test.go vendored
View File

@ -1,9 +0,0 @@
package cache
import (
"testing"
)
func TestMain(t *testing.T) {
}

4
src/ui/service/notification.go
View File

@ -25,7 +25,6 @@ import (
"github.com/vmware/harbor/src/common/utils"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/ui/api"
"github.com/vmware/harbor/src/ui/service/cache"
"github.com/astaxie/beego"
)
@ -82,9 +81,6 @@ func (n *NotificationHandler) Post() {
if err := dao.AddRepository(repoRecord); err != nil {
log.Errorf("Error happens when adding repository: %v", err)
}
if err := cache.RefreshCatalogCache(); err != nil {
log.Errorf("failed to refresh cache: %v", err)
}
}()
go api.TriggerReplicationByRepository(repository, []string{tag}, models.RepOpTransfer)
}

6
tests/apitests/apilib/search.go
View File

@ -22,12 +22,14 @@
package apilib
import()
import (
"github.com/vmware/harbor/src/common/models"
)
type Search struct {
// Search results of the projects that matched the filter keywords.
Projects []SearchProject `json:"project,omitempty"`
Projects []models.Project `json:"project,omitempty"`
// Search results of the repositories that matched the filter keywords.
Repositories []SearchRepository `json:"repository,omitempty"`