Merge pull request #5829 from steven-zou/support_chart_search

Enhance the global search API to include chart searching result

docs/swagger.yaml

@@ -21,15 +21,11 @@ security:
 paths:
   /search:
     get:
-      summary: Search for projects and repositories
+      summary: Search for projects, repositories and helm charts
       description: >
-        The Search endpoint returns information about the projects and
-        repositories
-
-        offered at public status or related to the current logged in user. The
-
-        response includes the project and repository list in a proper
-
+        The Search endpoint returns information about the projects ,repositories 
+        and helm charts offered at public status or related to the current logged in user. The
+        response includes the project, repository list and charts in a proper
         display order.
       parameters:
         - name: q
@@ -3075,16 +3071,21 @@ definitions:
   Search:
     type: object
     properties:
-      projects:
+      project:
         description: Search results of the projects that matched the filter keywords.
         type: array
         items:
           $ref: '#/definitions/Project'
-      repositories:
+      repository:
         description: Search results of the repositories that matched the filter keywords.
         type: array
         items:
           $ref: '#/definitions/SearchRepository'
+      chart:
+        description: Search results of the charts that macthed the filter keywords.
+        type: array
+        items:
+          $ref: '#/definitions/SearchResult'
   SearchRepository:
     type: object
     properties:
@@ -4481,6 +4482,19 @@ definitions:
         type: integer
         format: int64
         description: 'The time offset with the UTC 00:00 in seconds.'
+  SearchResult:
+    type: object
+    description: The chart search result item
+    properties:
+      name:
+        type: string
+        description: The chart name with repo name
+      score:
+        type: integer
+        description: The matched level
+      chart:
+        $ref: '#/definitions/ChartVersion'
+
         
         
     

src/chartserver/controller_test.go

@@ -185,6 +185,21 @@ func TestResponseRewrite(t *testing.T) {
 	}
 }
 
+// Test the chart searching
+func TestChartSearching(t *testing.T) {
+	namespaces := []string{"repo1", "repo2"}
+	q := "harbor"
+
+	results, err := mockController.GetRepositoryHandler().SearchChart(q, namespaces)
+	if err != nil {
+		t.Fatalf("expect nil error but got '%s'", err)
+	}
+
+	if len(results) != 2 {
+		t.Fatalf("expect 2 results but got %d", len(results))
+	}
+}
+
 // Clear environments
 func TestStopServers(t *testing.T) {
 	stopMockServers()

src/chartserver/repo_handler.go

@@ -2,17 +2,16 @@ package chartserver
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
 	"path"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/ghodss/yaml"
 	"github.com/goharbor/harbor/src/ui/filter"
+	"k8s.io/helm/cmd/helm/search"
 	helm_repo "k8s.io/helm/pkg/repo"
 
 	hlog "github.com/goharbor/harbor/src/common/utils/log"
@@ -20,6 +19,9 @@ import (
 
 const (
 	maxWorkers = 10
+
+	//Keep consistent with 'helm search' command
+	searchMaxScore = 25
 )
 
 // RepositoryHandler defines all the handlers to handle the requests related with chart repository
@@ -36,12 +38,6 @@ type RepositoryHandler struct {
 	backendServerAddress *url.URL
 }
 
-// Pass work to the workers
-// 'index' is the location of processing namespace/project in the list
-type workload struct {
-	index uint32
-}
-
 // Result returned by worker
 type processedResult struct {
 	namespace       string
@@ -77,122 +73,17 @@ func (rh *RepositoryHandler) GetIndexFile(w http.ResponseWriter, req *http.Reque
 		return
 	}
 
-	// The final merged index file
-	mergedIndexFile := &helm_repo.IndexFile{
-		APIVersion: "v1",
-		Entries:    make(map[string]helm_repo.ChartVersions),
-		Generated:  time.Now(),
-		PublicKeys: []string{},
+	namespaces := []string{}
+	for _, p := range results.Projects {
+		namespaces = append(namespaces, p.Name)
 	}
 
-	// Retrieve index.yaml for repositories
-	workerPool := make(chan *workload, maxWorkers)
-	// Sync the output results from the retriever
-	resultChan := make(chan *processedResult, 1)
-	// Receive error
-	errorChan := make(chan error, 1)
-	// Signal chan for merging work
-	mergeDone := make(chan struct{}, 1)
-	// Total projects/namespaces
-	total := uint32(results.Total)
-	// Track all the background threads
-	waitGroup := new(sync.WaitGroup)
-
-	// Initialize
-	initialItemCount := maxWorkers
-	if total < maxWorkers {
-		initialItemCount = int(total)
-	}
-	for i := 0; i < initialItemCount; i++ {
-		workerPool <- &workload{uint32(i)}
-	}
-
-	// Atomtic index
-	var indexRef uint32
-	atomic.AddUint32(&indexRef, uint32(initialItemCount-1))
-
-	// Start the index files merging thread
-	go func() {
-		defer func() {
-			mergeDone <- struct{}{}
-		}()
-
-		for res := range resultChan {
-			rh.mergeIndexFile(res.namespace, mergedIndexFile, res.indexFileOfRepo)
-		}
-	}()
-
-	// Retrieve the index files for the repositories
-	// and blocking here
-LOOP:
-	for {
-		select {
-		case work := <-workerPool:
-			if work.index >= total {
-				break LOOP
-			}
-			// Process
-			// New one
-			waitGroup.Add(1)
-			namespace := results.Projects[work.index].Name
-			go func(ns string) {
-				// Return the worker back to the pool
-				defer func() {
-					waitGroup.Done() // done
-
-					// Put one. The invalid index will be treated as a signal to quit loop
-					nextOne := atomic.AddUint32(&indexRef, 1)
-					workerPool <- &workload{nextOne}
-				}()
-
-				indexFile, err := rh.getIndexYamlWithNS(ns)
-				if err != nil {
-					errorChan <- err
-					return
-				}
-
-				// Output
-				resultChan <- &processedResult{
-					namespace:       ns,
-					indexFileOfRepo: indexFile,
-				}
-			}(namespace)
-		case err = <-errorChan:
-			// Quit earlier
-			break LOOP
-		case <-req.Context().Done():
-			// Quit earlier
-			err = errors.New("request of getting index yaml file is aborted")
-			break LOOP
-		}
-	}
-
-	// Hold util all the retrieving work are done
-	waitGroup.Wait()
-
-	// close consumer channel
-	close(resultChan)
-
-	// Wait until merging thread quit
-	<-mergeDone
-
-	// All the threads are done
-	// Met an error
+	mergedIndexFile, err := rh.getIndexYaml(namespaces)
 	if err != nil {
 		WriteInternalError(w, err)
 		return
 	}
 
-	// Remove duplicated keys in public key list
-	hash := make(map[string]string)
-	for _, key := range mergedIndexFile.PublicKeys {
-		hash[key] = key
-	}
-	mergedIndexFile.PublicKeys = []string{}
-	for k := range hash {
-		mergedIndexFile.PublicKeys = append(mergedIndexFile.PublicKeys, k)
-	}
-
 	bytes, err := yaml.Marshal(mergedIndexFile)
 	if err != nil {
 		WriteInternalError(w, err)
@@ -208,6 +99,160 @@ func (rh *RepositoryHandler) DownloadChartObject(w http.ResponseWriter, req *htt
 	rh.trafficProxy.ServeHTTP(w, req)
 }
 
+// SearchChart search charts in the specified namespaces with the keyword q.
+// RegExp mode is enabled as default.
+// For each chart, only the latest version will shown in the result list if matched to avoid duplicated entries.
+// Keep consistent with `helm search` command.
+func (rh *RepositoryHandler) SearchChart(q string, namespaces []string) ([]*search.Result, error) {
+	if len(q) == 0 || len(namespaces) == 0 {
+		// Return empty list
+		return []*search.Result{}, nil
+	}
+
+	// Get the merged index yaml file of the namespaces
+	ind, err := rh.getIndexYaml(namespaces)
+	if err != nil {
+		return nil, err
+	}
+
+	// Build the search index
+	index := search.NewIndex()
+	// As the repo name is already merged into the index yaml, we use empty repo name.
+	// Set 'All' to false to return only one version for each chart.
+	index.AddRepo("", ind, false)
+
+	// Search
+	// RegExp is enabled
+	results, err := index.Search(q, searchMaxScore, true)
+	if err != nil {
+		return nil, err
+	}
+
+	// Sort results.
+	search.SortScore(results)
+
+	return results, nil
+}
+
+// getIndexYaml will get the index yaml files for all the namespaces and merge them
+// as one unified index yaml file.
+func (rh *RepositoryHandler) getIndexYaml(namespaces []string) (*helm_repo.IndexFile, error) {
+	// The final merged index file
+	mergedIndexFile := &helm_repo.IndexFile{
+		APIVersion: "v1",
+		Entries:    make(map[string]helm_repo.ChartVersions),
+		Generated:  time.Now(),
+		PublicKeys: []string{},
+	}
+
+	// Sync the output results from the retriever
+	resultChan := make(chan *processedResult, 1)
+	// Receive error
+	errorChan := make(chan error, 1)
+	// Signal chan for merging work
+	mergeDone := make(chan struct{}, 1)
+	// Total projects/namespaces
+	total := len(namespaces)
+	// Initialize
+	initialItemCount := maxWorkers
+	if total < maxWorkers {
+		initialItemCount = total
+	}
+	// Retrieve index.yaml for repositories
+	workerPool := make(chan struct{}, initialItemCount)
+
+	// Add initial tokens to the pool
+	for i := 0; i < initialItemCount; i++ {
+		workerPool <- struct{}{}
+	}
+	// Track all the background threads
+	waitGroup := new(sync.WaitGroup)
+
+	// Start the index files merging thread
+	go func() {
+		defer func() {
+			mergeDone <- struct{}{}
+		}()
+
+		for res := range resultChan {
+			rh.mergeIndexFile(res.namespace, mergedIndexFile, res.indexFileOfRepo)
+		}
+	}()
+
+	// Retrieve the index files for the repositories
+	// and blocking here
+	var err error
+LOOP:
+	for _, ns := range namespaces {
+		// Check if error has occurred in some goroutines
+		select {
+		case err = <-errorChan:
+			break LOOP
+		default:
+			// do nothing
+		}
+
+		// Apply one token before processing
+		<-workerPool
+
+		waitGroup.Add(1)
+		go func(ns string) {
+			defer func() {
+				waitGroup.Done() //done
+				//Return the worker back to the pool
+				workerPool <- struct{}{}
+			}()
+
+			indexFile, err := rh.getIndexYamlWithNS(ns)
+			if err != nil {
+				if len(errorChan) == 0 {
+					//Only need one error as failure signal
+					errorChan <- err
+				}
+				return
+			}
+
+			// Output
+			resultChan <- &processedResult{
+				namespace:       ns,
+				indexFileOfRepo: indexFile,
+			}
+		}(ns)
+	}
+
+	// Hold util all the retrieving work are done
+	waitGroup.Wait()
+
+	// close merge channel
+	close(resultChan)
+
+	// Wait until merging thread quit
+	<-mergeDone
+
+	// All the threads are done
+	// Make sure error in the chan is read
+	if err == nil && len(errorChan) > 0 {
+		err = <-errorChan
+	}
+
+	// Met an error
+	if err != nil {
+		return nil, err
+	}
+
+	// Remove duplicated keys in public key list
+	hash := make(map[string]string)
+	for _, key := range mergedIndexFile.PublicKeys {
+		hash[key] = key
+	}
+	mergedIndexFile.PublicKeys = []string{}
+	for k := range hash {
+		mergedIndexFile.PublicKeys = append(mergedIndexFile.PublicKeys, k)
+	}
+
+	return mergedIndexFile, nil
+}
+
 // Get the index yaml file under the specified namespace from the backend server
 func (rh *RepositoryHandler) getIndexYamlWithNS(namespace string) (*helm_repo.IndexFile, error) {
 	// Join url path

src/ui/api/search.go

@@ -25,8 +25,13 @@ import (
 	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	uiutils "github.com/goharbor/harbor/src/ui/utils"
+	"k8s.io/helm/cmd/helm/search"
 )
 
+type chartSearchHandler func(string, []string) ([]*search.Result, error)
+
+var searchHandler chartSearchHandler
+
 // SearchAPI handles requesst to /api/search
 type SearchAPI struct {
 	BaseController
@@ -35,6 +40,7 @@ type SearchAPI struct {
 type searchResult struct {
 	Project    []*models.Project        `json:"project"`
 	Repository []map[string]interface{} `json:"repository"`
+	Chart      []*search.Result
 }
 
 // Get ...
@@ -80,7 +86,10 @@ func (s *SearchAPI) Get() {
 	}
 
 	projectResult := []*models.Project{}
+	proNames := []string{}
 	for _, p := range projects {
+		proNames = append(proNames, p.Name)
+
 		if len(keyword) > 0 && !strings.Contains(p.Name, keyword) {
 			continue
 		}
@@ -115,7 +124,21 @@ func (s *SearchAPI) Get() {
 		s.CustomAbort(http.StatusInternalServerError, "")
 	}
 
-	result := &searchResult{Project: projectResult, Repository: repositoryResult}
+	if searchHandler == nil {
+		searchHandler = chartController.GetRepositoryHandler().SearchChart
+	}
+
+	chartResults, err := searchHandler(keyword, proNames)
+	if err != nil {
+		log.Errorf("failed to filter charts: %v", err)
+		s.CustomAbort(http.StatusInternalServerError, err.Error())
+	}
+
+	result := &searchResult{
+		Project:    projectResult,
+		Repository: repositoryResult,
+		Chart:      chartResults,
+	}
 	s.Data["json"] = result
 	s.ServeJSON()
 }

src/ui/api/search_test.go

@@ -20,15 +20,28 @@ import (
 
 	"github.com/goharbor/harbor/src/common"
 	"github.com/goharbor/harbor/src/common/models"
+	"k8s.io/helm/cmd/helm/search"
 
 	"github.com/goharbor/harbor/src/common/dao"
 	member "github.com/goharbor/harbor/src/common/dao/project"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	helm_repo "k8s.io/helm/pkg/repo"
 )
 
 func TestSearch(t *testing.T) {
 	fmt.Println("Testing Search(SearchGet) API")
+	//Use mock chart search handler
+	searchHandler = func(string, []string) ([]*search.Result, error) {
+		results := []*search.Result{}
+		results = append(results, &search.Result{
+			Name:  "library/harbor",
+			Score: 0,
+			Chart: &helm_repo.ChartVersion{},
+		})
+
+		return results, nil
+	}
 	// create a public project named "search"
 	projectID1, err := dao.AddProject(models.Project{
 		Name:    "search",
@@ -164,4 +177,22 @@ func TestSearch(t *testing.T) {
 	assert.True(t, exist)
 	_, exist = repositories["search-2/hello-world"]
 	assert.True(t, exist)
+
+	//Search chart
+	err = handleAndParse(&testingRequest{
+		method: http.MethodGet,
+		url:    "/api/search",
+		queryStruct: struct {
+			Keyword string `url:"q"`
+		}{
+			Keyword: "harbor",
+		},
+		credential: sysAdmin,
+	}, result)
+	require.Nil(t, err)
+	require.Equal(t, 1, len(result.Chart))
+	require.Equal(t, "library/harbor", result.Chart[0].Name)
+
+	//Restore chart search handler
+	searchHandler = nil
 }