Merge pull request #1518 from ywk253100/170306_search

Refactor search API to return more info

docs/swagger.yaml

@@ -1508,26 +1508,12 @@ definitions:
         description: Search results of the projects that matched the filter keywords.
         type: array
         items:
-          $ref: '#/definitions/SearchProject'
+          $ref: '#/definitions/Project'
       repositories:
         description: Search results of the repositories that matched the filter keywords.
         type: array
         items:
           $ref: '#/definitions/SearchRepository'
-  SearchProject:
-    type: object
-    properties:
-      id:
-        type: integer
-        format: int64
-        description: The ID of project
-      name:
-        type: string
-        description: The name of the project
-      public:
-        type: integer
-        format: int
-        description: The flag to indicate the publicity of the project (1 is public, 0 is non-public)
   SearchRepository:
     type: object
     properties:
@@ -1543,6 +1529,12 @@ definitions:
       repository_name:
         type: string
         description: The name of the repository
+      pull_count:
+        type: integer
+        description: The count how many times the repository is pulled
+      tags_count:
+        type: integer
+        description: The count of tags in the repository		
   ProjectReq:
     type: object
     properties:

src/common/dao/project.go

@@ -168,10 +168,15 @@ func ToggleProjectPublicity(projectID int64, publicity int) error {
 // 2. the prject is public or the user is a member of the project
 func SearchProjects(userID int) ([]models.Project, error) {
 	o := GetOrmer()
-	sql := `select distinct p.project_id, p.name, p.public 
+
+	sql :=
+		`select distinct p.project_id, p.name, p.public, 
+			p.owner_id, p.creation_time, p.update_time, pm.role role
 		from project p 
-		left join project_member pm on p.project_id = pm.project_id 
+		left join project_member pm 
-		where (pm.user_id = ? or p.public = 1) and p.deleted = 0`
+		on p.project_id = pm.project_id 
+		where (pm.user_id = ? or p.public = 1) 
+		and p.deleted = 0 `
 
 	var projects []models.Project
 

src/common/dao/repository.go

@@ -50,7 +50,8 @@ func GetRepositoryByName(name string) (*models.RepoRecord, error) {
 func GetAllRepositories() ([]models.RepoRecord, error) {
 	o := GetOrmer()
 	var repos []models.RepoRecord
-	_, err := o.QueryTable("repository").All(&repos)
+	_, err := o.QueryTable("repository").
+		OrderBy("Name").All(&repos)
 	return repos, err
 }
 

src/ui/api/repository.go

@@ -28,7 +28,6 @@ import (
 	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/common/utils/registry"
-	"github.com/vmware/harbor/src/ui/service/cache"
 	svc_utils "github.com/vmware/harbor/src/ui/service/utils"
 
 	registry_error "github.com/vmware/harbor/src/common/utils/registry/error"
@@ -195,13 +194,6 @@ func (ra *RepositoryAPI) Delete() {
 			ra.CustomAbort(http.StatusInternalServerError, "")
 		}
 	}
-
-	go func() {
-		log.Debug("refreshing catalog cache")
-		if err := cache.RefreshCatalogCache(); err != nil {
-			log.Errorf("error occurred while refresh catalog cache: %v", err)
-		}
-	}()
 }
 
 type tag struct {
@@ -234,36 +226,46 @@ func (ra *RepositoryAPI) GetTags() {
 		}
 	}
 
-	rc, err := ra.initRepositoryClient(repoName)
+	client, err := ra.initRepositoryClient(repoName)
 	if err != nil {
 		log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
 		ra.CustomAbort(http.StatusInternalServerError, "internal error")
 	}
-
+	tags, err := listTag(client)
-	tags := []string{}
-
-	ts, err := rc.ListTag()
 	if err != nil {
 		regErr, ok := err.(*registry_error.Error)
 		if !ok {
 			log.Errorf("error occurred while listing tags of %s: %v", repoName, err)
 			ra.CustomAbort(http.StatusInternalServerError, "internal error")
 		}
+
+		ra.CustomAbort(regErr.StatusCode, regErr.Detail)
+	}
+
+	ra.Data["json"] = tags
+	ra.ServeJSON()
+}
+
+func listTag(client *registry.Repository) ([]string, error) {
+	tags := []string{}
+
+	ts, err := client.ListTag()
+	if err != nil {
 		// TODO remove the logic if the bug of registry is fixed
 		// It's a workaround for a bug of registry: when listing tags of
 		// a repository which is being pushed, a "NAME_UNKNOWN" error will
 		// been returned, while the catalog API can list this repository.
-		if regErr.StatusCode != http.StatusNotFound {
+
-			ra.CustomAbort(regErr.StatusCode, regErr.Detail)
+		if regErr, ok := err.(*registry_error.Error); ok &&
+			regErr.StatusCode == http.StatusNotFound {
+			return tags, nil
 		}
 	}
 
 	tags = append(tags, ts...)
-
 	sort.Strings(tags)
 
-	ra.Data["json"] = tags
+	return tags, nil
-	ra.ServeJSON()
 }
 
 // GetManifests handles GET /api/repositories/manifests
@@ -382,7 +384,7 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo
 		return nil, err
 	}
 
-	return cache.NewRepositoryClient(endpoint, !verify, username, repoName,
+	return NewRepositoryClient(endpoint, !verify, username, repoName,
 		"repository", repoName, "pull", "push", "*")
 }
 

src/ui/api/search.go

@@ -20,12 +20,12 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/vmware/harbor/src/common/api"
 	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
-	"github.com/vmware/harbor/src/ui/service/cache"
 	"github.com/vmware/harbor/src/common/utils"
 	"github.com/vmware/harbor/src/common/utils/log"
-    "github.com/vmware/harbor/src/common/api"
+	"github.com/vmware/harbor/src/ui/config"
 )
 
 // SearchAPI handles requesst to /api/search
@@ -34,7 +34,7 @@ type SearchAPI struct {
 }
 
 type searchResult struct {
-	Project    []map[string]interface{} `json:"project"`
+	Project    []models.Project         `json:"project"`
 	Repository []map[string]interface{} `json:"repository"`
 }
 
@@ -71,58 +71,104 @@ func (s *SearchAPI) Get() {
 
 	projectSorter := &models.ProjectSorter{Projects: projects}
 	sort.Sort(projectSorter)
-	projectResult := []map[string]interface{}{}
+	projectResult := []models.Project{}
 	for _, p := range projects {
 		match := true
 		if len(keyword) > 0 && !strings.Contains(p.Name, keyword) {
 			match = false
 		}
 		if match {
-			entry := make(map[string]interface{})
+			if userID != dao.NonExistUserID {
-			entry["id"] = p.ProjectID
+				if isSysAdmin {
-			entry["name"] = p.Name
+					p.Role = models.PROJECTADMIN
-			entry["public"] = p.Public
+				}
-			projectResult = append(projectResult, entry)
+				if p.Role == models.PROJECTADMIN {
+					p.Togglable = true
+				}
+			}
+
+			repos, err := dao.GetRepositoryByProjectName(p.Name)
+			if err != nil {
+				log.Errorf("failed to get repositories of project %s: %v", p.Name, err)
+				s.CustomAbort(http.StatusInternalServerError, "")
+			}
+
+			p.RepoCount = len(repos)
+
+			projectResult = append(projectResult, p)
 		}
 	}
 
-	repositories, err := cache.GetRepoFromCache()
+	repositoryResult, err := filterRepositories(projects, keyword)
 	if err != nil {
-		log.Errorf("failed to list repositories: %v", err)
+		log.Errorf("failed to filter repositories: %v", err)
 		s.CustomAbort(http.StatusInternalServerError, "")
 	}
 
-	sort.Strings(repositories)
-	repositoryResult := filterRepositories(repositories, projects, keyword)
 	result := &searchResult{Project: projectResult, Repository: repositoryResult}
 	s.Data["json"] = result
 	s.ServeJSON()
 }
 
-func filterRepositories(repositories []string, projects []models.Project, keyword string) []map[string]interface{} {
+func filterRepositories(projects []models.Project, keyword string) (
+	[]map[string]interface{}, error) {
+
+	repositories, err := dao.GetAllRepositories()
+	if err != nil {
+		return nil, err
+	}
+
 	i, j := 0, 0
 	result := []map[string]interface{}{}
 	for i < len(repositories) && j < len(projects) {
 		r := repositories[i]
-		p, _ := utils.ParseRepository(r)
+		p, _ := utils.ParseRepository(r.Name)
 		d := strings.Compare(p, projects[j].Name)
 		if d < 0 {
 			i++
 			continue
 		} else if d == 0 {
 			i++
-			if len(keyword) != 0 && !strings.Contains(r, keyword) {
+			if len(keyword) != 0 && !strings.Contains(r.Name, keyword) {
 				continue
 			}
 			entry := make(map[string]interface{})
-			entry["repository_name"] = r
+			entry["repository_name"] = r.Name
 			entry["project_name"] = projects[j].Name
 			entry["project_id"] = projects[j].ProjectID
 			entry["project_public"] = projects[j].Public
+			entry["pull_count"] = r.PullCount
+
+			tags, err := getTags(r.Name)
+			if err != nil {
+				return nil, err
+			}
+			entry["tags_count"] = len(tags)
+
 			result = append(result, entry)
 		} else {
 			j++
 		}
 	}
-	return result
+	return result, nil
+}
+
+func getTags(repository string) ([]string, error) {
+	url, err := config.RegistryURL()
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := NewRepositoryClient(url, true,
+		"admin", repository, "repository", repository, "pull")
+	if err != nil {
+		return nil, err
+	}
+
+	tags, err := listTag(client)
+	if err != nil {
+		return nil, err
+	}
+
+	return tags, nil
 }

src/ui/api/search_test.go

@@ -22,9 +22,9 @@ func TestSearch(t *testing.T) {
 		t.Log(err)
 	} else {
 		assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
-		assert.Equal(int64(1), result.Projects[0].Id, "Project id should be equal")
+		assert.Equal(int64(1), result.Projects[0].ProjectID, "Project id should be equal")
 		assert.Equal("library", result.Projects[0].Name, "Project name should be library")
-		assert.Equal(int32(1), result.Projects[0].Public, "Project public status should be 1 (true)")
+		assert.Equal(1, result.Projects[0].Public, "Project public status should be 1 (true)")
 	}
 
 	//--------case 2 : Response Code  = 200, sysAdmin and search repo--------//

src/ui/api/utils.go

@@ -29,9 +29,9 @@ import (
 	"github.com/vmware/harbor/src/common/utils"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/common/utils/registry"
+	"github.com/vmware/harbor/src/common/utils/registry/auth"
 	registry_error "github.com/vmware/harbor/src/common/utils/registry/error"
 	"github.com/vmware/harbor/src/ui/config"
-	"github.com/vmware/harbor/src/ui/service/cache"
 )
 
 func checkProjectPermission(userID int, projectID int64) bool {
@@ -352,7 +352,7 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string
 			if err != nil {
 				return needsAdd, needsDel, err
 			}
-			client, err := cache.NewRepositoryClient(endpoint, true,
+			client, err := NewRepositoryClient(endpoint, true,
 				"admin", repoInR, "repository", repoInR)
 			if err != nil {
 				return needsAdd, needsDel, err
@@ -377,7 +377,7 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string
 			if err != nil {
 				return needsAdd, needsDel, err
 			}
-			client, err := cache.NewRepositoryClient(endpoint, true,
+			client, err := NewRepositoryClient(endpoint, true,
 				"admin", repoInR, "repository", repoInR)
 			if err != nil {
 				return needsAdd, needsDel, err
@@ -440,7 +440,7 @@ func initRegistryClient() (r *registry.Registry, err error) {
 		return nil, err
 	}
 
-	registryClient, err := cache.NewRegistryClient(endpoint, true, "admin",
+	registryClient, err := NewRegistryClient(endpoint, true, "admin",
 		"registry", "catalog", "*")
 	if err != nil {
 		return nil, err
@@ -485,10 +485,6 @@ func getReposByProject(name string, keyword ...string) ([]string, error) {
 	return repositories, nil
 }
 
-func getAllRepos() ([]string, error) {
-	return cache.GetRepoFromCache()
-}
-
 func repositoryExist(name string, client *registry.Repository) (bool, error) {
 	tags, err := client.ListTag()
 	if err != nil {
@@ -499,3 +495,38 @@ func repositoryExist(name string, client *registry.Repository) (bool, error) {
 	}
 	return len(tags) != 0, nil
 }
+
+// NewRegistryClient ...
+func NewRegistryClient(endpoint string, insecure bool, username, scopeType, scopeName string,
+	scopeActions ...string) (*registry.Registry, error) {
+	authorizer := auth.NewUsernameTokenAuthorizer(username, scopeType, scopeName, scopeActions...)
+
+	store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := registry.NewRegistryWithModifiers(endpoint, insecure, store)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
+// NewRepositoryClient ...
+func NewRepositoryClient(endpoint string, insecure bool, username, repository, scopeType, scopeName string,
+	scopeActions ...string) (*registry.Repository, error) {
+
+	authorizer := auth.NewUsernameTokenAuthorizer(username, scopeType, scopeName, scopeActions...)
+
+	store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}

src/ui/service/cache/cache.go

@@ -1,119 +0,0 @@
-/*
-   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package cache
-
-import (
-	"time"
-
-	"github.com/vmware/harbor/src/common/dao"
-	"github.com/vmware/harbor/src/common/utils/log"
-	"github.com/vmware/harbor/src/common/utils/registry"
-	"github.com/vmware/harbor/src/common/utils/registry/auth"
-
-	"github.com/astaxie/beego/cache"
-)
-
-var (
-	// Cache is the global cache in system.
-	Cache cache.Cache
-)
-
-const catalogKey string = "catalog"
-
-func init() {
-	var err error
-	Cache, err = cache.NewCache("memory", `{"interval":720}`)
-	if err != nil {
-		log.Errorf("Failed to initialize cache, error:%v", err)
-	}
-}
-
-// RefreshCatalogCache calls registry's API to get repository list and write it to cache.
-func RefreshCatalogCache() error {
-	log.Debug("refreshing catalog cache...")
-
-	repos, err := getAllRepositories()
-	if err != nil {
-		return err
-	}
-	Cache.Put(catalogKey, repos, 600*time.Second)
-	return nil
-}
-
-// GetRepoFromCache get repository list from cache, it refreshes the cache if it's empty.
-func GetRepoFromCache() ([]string, error) {
-
-	result := Cache.Get(catalogKey)
-	if result == nil {
-		err := RefreshCatalogCache()
-		if err != nil {
-			return nil, err
-		}
-		cached := Cache.Get(catalogKey)
-		if cached != nil {
-			return cached.([]string), nil
-		}
-		return nil, nil
-	}
-	return result.([]string), nil
-}
-
-func getAllRepositories() ([]string, error) {
-	var repos []string
-	rs, err := dao.GetAllRepositories()
-	if err != nil {
-		return repos, err
-	}
-	for _, e := range rs {
-		repos = append(repos, e.Name)
-	}
-	return repos, nil
-}
-
-// NewRegistryClient ...
-func NewRegistryClient(endpoint string, insecure bool, username, scopeType, scopeName string,
-	scopeActions ...string) (*registry.Registry, error) {
-	authorizer := auth.NewUsernameTokenAuthorizer(username, scopeType, scopeName, scopeActions...)
-
-	store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
-	if err != nil {
-		return nil, err
-	}
-
-	client, err := registry.NewRegistryWithModifiers(endpoint, insecure, store)
-	if err != nil {
-		return nil, err
-	}
-	return client, nil
-}
-
-// NewRepositoryClient ...
-func NewRepositoryClient(endpoint string, insecure bool, username, repository, scopeType, scopeName string,
-	scopeActions ...string) (*registry.Repository, error) {
-
-	authorizer := auth.NewUsernameTokenAuthorizer(username, scopeType, scopeName, scopeActions...)
-
-	store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
-	if err != nil {
-		return nil, err
-	}
-
-	client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store)
-	if err != nil {
-		return nil, err
-	}
-	return client, nil
-}

src/ui/service/cache/cache_test.go

@@ -1,9 +0,0 @@
-package cache
-
-import (
-	"testing"
-)
-
-func TestMain(t *testing.T) {
-}
-

src/ui/service/notification.go

@@ -25,7 +25,6 @@ import (
 	"github.com/vmware/harbor/src/common/utils"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/ui/api"
-	"github.com/vmware/harbor/src/ui/service/cache"
 
 	"github.com/astaxie/beego"
 )
@@ -82,9 +81,6 @@ func (n *NotificationHandler) Post() {
 				if err := dao.AddRepository(repoRecord); err != nil {
 					log.Errorf("Error happens when adding repository: %v", err)
 				}
-				if err := cache.RefreshCatalogCache(); err != nil {
-					log.Errorf("failed to refresh cache: %v", err)
-				}
 			}()
 			go api.TriggerReplicationByRepository(repository, []string{tag}, models.RepOpTransfer)
 		}

tests/apitests/apilib/search.go

@@ -1,10 +1,10 @@
-/* 
+/*
  * Harbor API
  *
  * These APIs provide services for manipulating Harbor project.
  *
  * OpenAPI spec version: 0.3.0
- * 
+ *
  * Generated by: https://github.com/swagger-api/swagger-codegen.git
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,12 +22,14 @@
 
 package apilib
 
-import()
+import (
+	"github.com/vmware/harbor/src/common/models"
+)
 
 type Search struct {
 
 	// Search results of the projects that matched the filter keywords.
-	Projects []SearchProject `json:"project,omitempty"`
+	Projects []models.Project `json:"project,omitempty"`
 
 	// Search results of the repositories that matched the filter keywords.
 	Repositories []SearchRepository `json:"repository,omitempty"`