From a9f2ff2c91fbaa1f4ca22052de04324a63ff7d80 Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Wed, 5 Sep 2018 20:13:27 +0800 Subject: [PATCH] Enhance the global search API to include chart searching result - add chart search interface method in utility handler - update the search API handler - update the search API UT case Signed-off-by: Steven Zou --- docs/swagger.yaml | 34 ++-- src/chartserver/controller_test.go | 15 ++ src/chartserver/repo_handler.go | 279 +++++++++++++++++------------ src/ui/api/search.go | 25 ++- src/ui/api/search_test.go | 31 ++++ 5 files changed, 256 insertions(+), 128 deletions(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 8b8ea6826..3b8a1e03f 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -21,15 +21,11 @@ security: paths: /search: get: - summary: Search for projects and repositories + summary: Search for projects, repositories and helm charts description: > - The Search endpoint returns information about the projects and - repositories - - offered at public status or related to the current logged in user. The - - response includes the project and repository list in a proper - + The Search endpoint returns information about the projects ,repositories + and helm charts offered at public status or related to the current logged in user. The + response includes the project, repository list and charts in a proper display order. parameters: - name: q @@ -3075,16 +3071,21 @@ definitions: Search: type: object properties: - projects: + project: description: Search results of the projects that matched the filter keywords. type: array items: $ref: '#/definitions/Project' - repositories: + repository: description: Search results of the repositories that matched the filter keywords. type: array items: $ref: '#/definitions/SearchRepository' + chart: + description: Search results of the charts that macthed the filter keywords. + type: array + items: + $ref: '#/definitions/SearchResult' SearchRepository: type: object properties: @@ -4481,6 +4482,19 @@ definitions: type: integer format: int64 description: 'The time offset with the UTC 00:00 in seconds.' + SearchResult: + type: object + description: The chart search result item + properties: + name: + type: string + description: The chart name with repo name + score: + type: integer + description: The matched level + chart: + $ref: '#/definitions/ChartVersion' + diff --git a/src/chartserver/controller_test.go b/src/chartserver/controller_test.go index 4a83ae4cb..0bbc52228 100644 --- a/src/chartserver/controller_test.go +++ b/src/chartserver/controller_test.go @@ -185,6 +185,21 @@ func TestResponseRewrite(t *testing.T) { } } +// Test the chart searching +func TestChartSearching(t *testing.T) { + namespaces := []string{"repo1", "repo2"} + q := "harbor" + + results, err := mockController.GetRepositoryHandler().SearchChart(q, namespaces) + if err != nil { + t.Fatalf("expect nil error but got '%s'", err) + } + + if len(results) != 2 { + t.Fatalf("expect 2 results but got %d", len(results)) + } +} + // Clear environments func TestStopServers(t *testing.T) { stopMockServers() diff --git a/src/chartserver/repo_handler.go b/src/chartserver/repo_handler.go index b640e3995..31330bc9e 100644 --- a/src/chartserver/repo_handler.go +++ b/src/chartserver/repo_handler.go @@ -2,17 +2,16 @@ package chartserver import ( "encoding/json" - "errors" "fmt" "net/http" "net/url" "path" "sync" - "sync/atomic" "time" "github.com/ghodss/yaml" "github.com/goharbor/harbor/src/ui/filter" + "k8s.io/helm/cmd/helm/search" helm_repo "k8s.io/helm/pkg/repo" hlog "github.com/goharbor/harbor/src/common/utils/log" @@ -20,6 +19,9 @@ import ( const ( maxWorkers = 10 + + //Keep consistent with 'helm search' command + searchMaxScore = 25 ) // RepositoryHandler defines all the handlers to handle the requests related with chart repository @@ -36,12 +38,6 @@ type RepositoryHandler struct { backendServerAddress *url.URL } -// Pass work to the workers -// 'index' is the location of processing namespace/project in the list -type workload struct { - index uint32 -} - // Result returned by worker type processedResult struct { namespace string @@ -77,122 +73,17 @@ func (rh *RepositoryHandler) GetIndexFile(w http.ResponseWriter, req *http.Reque return } - // The final merged index file - mergedIndexFile := &helm_repo.IndexFile{ - APIVersion: "v1", - Entries: make(map[string]helm_repo.ChartVersions), - Generated: time.Now(), - PublicKeys: []string{}, + namespaces := []string{} + for _, p := range results.Projects { + namespaces = append(namespaces, p.Name) } - // Retrieve index.yaml for repositories - workerPool := make(chan *workload, maxWorkers) - // Sync the output results from the retriever - resultChan := make(chan *processedResult, 1) - // Receive error - errorChan := make(chan error, 1) - // Signal chan for merging work - mergeDone := make(chan struct{}, 1) - // Total projects/namespaces - total := uint32(results.Total) - // Track all the background threads - waitGroup := new(sync.WaitGroup) - - // Initialize - initialItemCount := maxWorkers - if total < maxWorkers { - initialItemCount = int(total) - } - for i := 0; i < initialItemCount; i++ { - workerPool <- &workload{uint32(i)} - } - - // Atomtic index - var indexRef uint32 - atomic.AddUint32(&indexRef, uint32(initialItemCount-1)) - - // Start the index files merging thread - go func() { - defer func() { - mergeDone <- struct{}{} - }() - - for res := range resultChan { - rh.mergeIndexFile(res.namespace, mergedIndexFile, res.indexFileOfRepo) - } - }() - - // Retrieve the index files for the repositories - // and blocking here -LOOP: - for { - select { - case work := <-workerPool: - if work.index >= total { - break LOOP - } - // Process - // New one - waitGroup.Add(1) - namespace := results.Projects[work.index].Name - go func(ns string) { - // Return the worker back to the pool - defer func() { - waitGroup.Done() // done - - // Put one. The invalid index will be treated as a signal to quit loop - nextOne := atomic.AddUint32(&indexRef, 1) - workerPool <- &workload{nextOne} - }() - - indexFile, err := rh.getIndexYamlWithNS(ns) - if err != nil { - errorChan <- err - return - } - - // Output - resultChan <- &processedResult{ - namespace: ns, - indexFileOfRepo: indexFile, - } - }(namespace) - case err = <-errorChan: - // Quit earlier - break LOOP - case <-req.Context().Done(): - // Quit earlier - err = errors.New("request of getting index yaml file is aborted") - break LOOP - } - } - - // Hold util all the retrieving work are done - waitGroup.Wait() - - // close consumer channel - close(resultChan) - - // Wait until merging thread quit - <-mergeDone - - // All the threads are done - // Met an error + mergedIndexFile, err := rh.getIndexYaml(namespaces) if err != nil { WriteInternalError(w, err) return } - // Remove duplicated keys in public key list - hash := make(map[string]string) - for _, key := range mergedIndexFile.PublicKeys { - hash[key] = key - } - mergedIndexFile.PublicKeys = []string{} - for k := range hash { - mergedIndexFile.PublicKeys = append(mergedIndexFile.PublicKeys, k) - } - bytes, err := yaml.Marshal(mergedIndexFile) if err != nil { WriteInternalError(w, err) @@ -208,6 +99,160 @@ func (rh *RepositoryHandler) DownloadChartObject(w http.ResponseWriter, req *htt rh.trafficProxy.ServeHTTP(w, req) } +// SearchChart search charts in the specified namespaces with the keyword q. +// RegExp mode is enabled as default. +// For each chart, only the latest version will shown in the result list if matched to avoid duplicated entries. +// Keep consistent with `helm search` command. +func (rh *RepositoryHandler) SearchChart(q string, namespaces []string) ([]*search.Result, error) { + if len(q) == 0 || len(namespaces) == 0 { + // Return empty list + return []*search.Result{}, nil + } + + // Get the merged index yaml file of the namespaces + ind, err := rh.getIndexYaml(namespaces) + if err != nil { + return nil, err + } + + // Build the search index + index := search.NewIndex() + // As the repo name is already merged into the index yaml, we use empty repo name. + // Set 'All' to false to return only one version for each chart. + index.AddRepo("", ind, false) + + // Search + // RegExp is enabled + results, err := index.Search(q, searchMaxScore, true) + if err != nil { + return nil, err + } + + // Sort results. + search.SortScore(results) + + return results, nil +} + +// getIndexYaml will get the index yaml files for all the namespaces and merge them +// as one unified index yaml file. +func (rh *RepositoryHandler) getIndexYaml(namespaces []string) (*helm_repo.IndexFile, error) { + // The final merged index file + mergedIndexFile := &helm_repo.IndexFile{ + APIVersion: "v1", + Entries: make(map[string]helm_repo.ChartVersions), + Generated: time.Now(), + PublicKeys: []string{}, + } + + // Sync the output results from the retriever + resultChan := make(chan *processedResult, 1) + // Receive error + errorChan := make(chan error, 1) + // Signal chan for merging work + mergeDone := make(chan struct{}, 1) + // Total projects/namespaces + total := len(namespaces) + // Initialize + initialItemCount := maxWorkers + if total < maxWorkers { + initialItemCount = total + } + // Retrieve index.yaml for repositories + workerPool := make(chan struct{}, initialItemCount) + + // Add initial tokens to the pool + for i := 0; i < initialItemCount; i++ { + workerPool <- struct{}{} + } + // Track all the background threads + waitGroup := new(sync.WaitGroup) + + // Start the index files merging thread + go func() { + defer func() { + mergeDone <- struct{}{} + }() + + for res := range resultChan { + rh.mergeIndexFile(res.namespace, mergedIndexFile, res.indexFileOfRepo) + } + }() + + // Retrieve the index files for the repositories + // and blocking here + var err error +LOOP: + for _, ns := range namespaces { + // Check if error has occurred in some goroutines + select { + case err = <-errorChan: + break LOOP + default: + // do nothing + } + + // Apply one token before processing + <-workerPool + + waitGroup.Add(1) + go func(ns string) { + defer func() { + waitGroup.Done() //done + //Return the worker back to the pool + workerPool <- struct{}{} + }() + + indexFile, err := rh.getIndexYamlWithNS(ns) + if err != nil { + if len(errorChan) == 0 { + //Only need one error as failure signal + errorChan <- err + } + return + } + + // Output + resultChan <- &processedResult{ + namespace: ns, + indexFileOfRepo: indexFile, + } + }(ns) + } + + // Hold util all the retrieving work are done + waitGroup.Wait() + + // close merge channel + close(resultChan) + + // Wait until merging thread quit + <-mergeDone + + // All the threads are done + // Make sure error in the chan is read + if err == nil && len(errorChan) > 0 { + err = <-errorChan + } + + // Met an error + if err != nil { + return nil, err + } + + // Remove duplicated keys in public key list + hash := make(map[string]string) + for _, key := range mergedIndexFile.PublicKeys { + hash[key] = key + } + mergedIndexFile.PublicKeys = []string{} + for k := range hash { + mergedIndexFile.PublicKeys = append(mergedIndexFile.PublicKeys, k) + } + + return mergedIndexFile, nil +} + // Get the index yaml file under the specified namespace from the backend server func (rh *RepositoryHandler) getIndexYamlWithNS(namespace string) (*helm_repo.IndexFile, error) { // Join url path diff --git a/src/ui/api/search.go b/src/ui/api/search.go index 9e2b887b1..957165be3 100644 --- a/src/ui/api/search.go +++ b/src/ui/api/search.go @@ -25,8 +25,13 @@ import ( "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/common/utils/log" uiutils "github.com/goharbor/harbor/src/ui/utils" + "k8s.io/helm/cmd/helm/search" ) +type chartSearchHandler func(string, []string) ([]*search.Result, error) + +var searchHandler chartSearchHandler + // SearchAPI handles requesst to /api/search type SearchAPI struct { BaseController @@ -35,6 +40,7 @@ type SearchAPI struct { type searchResult struct { Project []*models.Project `json:"project"` Repository []map[string]interface{} `json:"repository"` + Chart []*search.Result } // Get ... @@ -80,7 +86,10 @@ func (s *SearchAPI) Get() { } projectResult := []*models.Project{} + proNames := []string{} for _, p := range projects { + proNames = append(proNames, p.Name) + if len(keyword) > 0 && !strings.Contains(p.Name, keyword) { continue } @@ -115,7 +124,21 @@ func (s *SearchAPI) Get() { s.CustomAbort(http.StatusInternalServerError, "") } - result := &searchResult{Project: projectResult, Repository: repositoryResult} + if searchHandler == nil { + searchHandler = chartController.GetRepositoryHandler().SearchChart + } + + chartResults, err := searchHandler(keyword, proNames) + if err != nil { + log.Errorf("failed to filter charts: %v", err) + s.CustomAbort(http.StatusInternalServerError, err.Error()) + } + + result := &searchResult{ + Project: projectResult, + Repository: repositoryResult, + Chart: chartResults, + } s.Data["json"] = result s.ServeJSON() } diff --git a/src/ui/api/search_test.go b/src/ui/api/search_test.go index a3e9535c6..ac3926e33 100644 --- a/src/ui/api/search_test.go +++ b/src/ui/api/search_test.go @@ -20,15 +20,28 @@ import ( "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/models" + "k8s.io/helm/cmd/helm/search" "github.com/goharbor/harbor/src/common/dao" member "github.com/goharbor/harbor/src/common/dao/project" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + helm_repo "k8s.io/helm/pkg/repo" ) func TestSearch(t *testing.T) { fmt.Println("Testing Search(SearchGet) API") + //Use mock chart search handler + searchHandler = func(string, []string) ([]*search.Result, error) { + results := []*search.Result{} + results = append(results, &search.Result{ + Name: "library/harbor", + Score: 0, + Chart: &helm_repo.ChartVersion{}, + }) + + return results, nil + } // create a public project named "search" projectID1, err := dao.AddProject(models.Project{ Name: "search", @@ -164,4 +177,22 @@ func TestSearch(t *testing.T) { assert.True(t, exist) _, exist = repositories["search-2/hello-world"] assert.True(t, exist) + + //Search chart + err = handleAndParse(&testingRequest{ + method: http.MethodGet, + url: "/api/search", + queryStruct: struct { + Keyword string `url:"q"` + }{ + Keyword: "harbor", + }, + credential: sysAdmin, + }, result) + require.Nil(t, err) + require.Equal(t, 1, len(result.Chart)) + require.Equal(t, "library/harbor", result.Chart[0].Name) + + //Restore chart search handler + searchHandler = nil }