From df241c0666610cc6de2585d681de2d160aa57ae4 Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Wed, 19 Sep 2018 14:45:52 +0800 Subject: [PATCH] Refactor the chart service implementation to provide more extemding flexibilities - use unified proxy handler to handle all the proxy traffic - define interfaces for the handlers - implement the interface at chart controller level - refactor UT cases - update other related reference code Signed-off-by: Steven Zou --- src/chartserver/base_handler.go | 17 -- src/chartserver/base_test.go | 261 +++++++----------- src/chartserver/chart_operator.go | 15 +- src/chartserver/controller.go | 118 ++------ src/chartserver/controller_test.go | 220 +-------------- src/chartserver/handler_interface.go | 89 ++++++ src/chartserver/handler_manipulation.go | 93 +++++++ src/chartserver/handler_manipulation_test.go | 88 ++++++ src/chartserver/handler_proxy_traffic.go | 12 + src/chartserver/handler_proxy_traffic_test.go | 136 +++++++++ .../{repo_handler.go => handler_repo.go} | 133 ++------- src/chartserver/handler_repo_test.go | 32 +++ src/chartserver/handler_utility.go | 208 ++++++++++++++ src/chartserver/handler_utility_test.go | 83 ++++++ src/chartserver/manipulation_handler.go | 199 ------------- src/chartserver/reverse_proxy.go | 37 +++ src/chartserver/utility_handler.go | 163 ----------- src/chartserver/utility_handler_test.go | 228 --------------- src/chartserver/utils.go | 61 ++-- src/ui/api/base.go | 25 ++ src/ui/api/chart_label.go | 2 +- src/ui/api/chart_repository.go | 199 ++++++++----- src/ui/api/chart_repository_test.go | 40 --- src/ui/api/project.go | 2 +- src/ui/api/search.go | 2 +- 25 files changed, 1141 insertions(+), 1322 deletions(-) delete mode 100644 src/chartserver/base_handler.go create mode 100644 src/chartserver/handler_interface.go create mode 100644 src/chartserver/handler_manipulation.go create mode 100644 src/chartserver/handler_manipulation_test.go create mode 100644 src/chartserver/handler_proxy_traffic.go create mode 100644 src/chartserver/handler_proxy_traffic_test.go rename src/chartserver/{repo_handler.go => handler_repo.go} (55%) create mode 100644 src/chartserver/handler_repo_test.go create mode 100644 src/chartserver/handler_utility.go create mode 100644 src/chartserver/handler_utility_test.go delete mode 100644 src/chartserver/manipulation_handler.go delete mode 100644 src/chartserver/utility_handler.go delete mode 100644 src/chartserver/utility_handler_test.go diff --git a/src/chartserver/base_handler.go b/src/chartserver/base_handler.go deleted file mode 100644 index 2cd55e2d1..000000000 --- a/src/chartserver/base_handler.go +++ /dev/null @@ -1,17 +0,0 @@ -package chartserver - -import ( - "net/http" -) - -// BaseHandler defines the handlers related with the chart server itself. -type BaseHandler struct { - // Proxy used to to transfer the traffic of requests - // It's mainly used to talk to the backend chart server - trafficProxy *ProxyEngine -} - -// GetHealthStatus will return the health status of the backend chart repository server -func (bh *BaseHandler) GetHealthStatus(w http.ResponseWriter, req *http.Request) { - bh.trafficProxy.ServeHTTP(w, req) -} diff --git a/src/chartserver/base_test.go b/src/chartserver/base_test.go index ab7bc4c72..6513a3f19 100644 --- a/src/chartserver/base_test.go +++ b/src/chartserver/base_test.go @@ -1,22 +1,14 @@ package chartserver import ( - "context" "encoding/json" - "errors" - "fmt" "net/http" "net/http/httptest" "net/url" - "regexp" - - "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/ui/filter" - "github.com/goharbor/harbor/src/ui/promgr/metamgr" ) -// The backend server -var mockServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// The backend server handler +var mockHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.RequestURI { case "/health": if r.Method == http.MethodGet { @@ -52,13 +44,27 @@ var mockServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.Respon w.Write([]byte(harborChartV)) return } + if r.Method == http.MethodDelete { + w.WriteHeader(http.StatusOK) + return + } case "/api/repo1/charts/harbor/1.0.0": if r.Method == http.MethodGet { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("Internal error")) + w.WriteHeader(http.StatusNotFound) + w.Write([]byte("Not found")) + return + } + case "/api/repo1/charts/harbor/0.2.1": + if r.Method == http.MethodDelete { + w.WriteHeader(http.StatusOK) return } case "/api/repo1/charts/harbor": + if r.Method == http.MethodGet { + w.Write([]byte(chartVersionsOfHarbor)) + return + } + case "/repo3/charts/harbor-0.8.1.tgz": if r.Method == http.MethodGet { w.WriteHeader(http.StatusUnauthorized) w.Write([]byte("Unauthorized")) @@ -68,164 +74,99 @@ var mockServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.Respon w.WriteHeader(http.StatusNotImplemented) w.Write([]byte("not supported")) -})) +}) -// Chart controller reference -var mockController *Controller - -// The frontend server -var frontServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.RequestURI == "/health" && r.Method == http.MethodGet { - mockController.GetBaseHandler().GetHealthStatus(w, r) - return +func createMockObjects() (*httptest.Server, *Controller, error) { + s := httptest.NewServer(mockHandler) + backendURL, err := url.Parse(s.URL) + if err != nil { + s.Close() + return nil, nil, err } - if r.RequestURI == "/index.yaml" && r.Method == http.MethodGet { - mockProjectMgr := &mockProjectManager{} - *r = *(r.WithContext(context.WithValue(r.Context(), filter.PmKey, mockProjectMgr))) - mockController.GetRepositoryHandler().GetIndexFile(w, r) - return + mockController, err := NewController(backendURL) + if err != nil { + s.Close() + return nil, nil, err } - var indexYaml = regexp.MustCompile(`^/\w+/index.yaml$`) - if r.Method == http.MethodGet && indexYaml.MatchString(r.RequestURI) { - mockController.GetRepositoryHandler().GetIndexFileWithNS(w, r) - return - } - - var downloadFile = regexp.MustCompile(`^/\w+/charts/.+$`) - if r.Method == http.MethodGet && downloadFile.MatchString(r.RequestURI) { - mockController.GetRepositoryHandler().DownloadChartObject(w, r) - return - } - - var listCharts = regexp.MustCompile(`^/api/\w+/charts$`) - if r.Method == http.MethodGet && listCharts.MatchString(r.RequestURI) { - mockController.GetManipulationHandler().ListCharts(w, r) - return - } - - var getChart = regexp.MustCompile(`^/api/\w+/charts/[\w-]+$`) - if r.Method == http.MethodGet && getChart.MatchString(r.RequestURI) { - mockController.GetManipulationHandler().GetChart(w, r) - return - } - - var getChartV = regexp.MustCompile(`^/api/\w+/charts/.+/.+$`) - if r.Method == http.MethodGet && getChartV.MatchString(r.RequestURI) { - *r = *(r.WithContext(context.WithValue(r.Context(), NamespaceContextKey, "repo1"))) - mockController.GetManipulationHandler().GetChartVersion(w, r) - return - } - - var postCharts = regexp.MustCompile(`^/api/\w+/charts$`) - if r.Method == http.MethodPost && postCharts.MatchString(r.RequestURI) { - mockController.GetManipulationHandler().UploadChartVersion(w, r) - return - } - - var postProv = regexp.MustCompile(`^/api/\w+/prov$`) - if r.Method == http.MethodPost && postProv.MatchString(r.RequestURI) { - mockController.GetManipulationHandler().UploadProvenanceFile(w, r) - return - } - - var deleteChartV = regexp.MustCompile(`^/api/\w+/charts/.+/.+$`) - if r.Method == http.MethodDelete && deleteChartV.MatchString(r.RequestURI) { - mockController.GetManipulationHandler().DeleteChartVersion(w, r) - return - } - - w.WriteHeader(http.StatusNotImplemented) - w.Write([]byte(fmt.Sprintf("no handler to handle %s", r.RequestURI))) -})) + return s, mockController, nil +} // Http client var httpClient = NewChartClient(nil) -// Mock project manager -type mockProjectManager struct{} - -func (mpm *mockProjectManager) Get(projectIDOrName interface{}) (*models.Project, error) { - return nil, errors.New("Not implemented") -} - -func (mpm *mockProjectManager) Create(*models.Project) (int64, error) { - return -1, errors.New("Not implemented") -} - -func (mpm *mockProjectManager) Delete(projectIDOrName interface{}) error { - return errors.New("Not implemented") -} - -func (mpm *mockProjectManager) Update(projectIDOrName interface{}, project *models.Project) error { - return errors.New("Not implemented") -} -func (mpm *mockProjectManager) List(query *models.ProjectQueryParam) (*models.ProjectQueryResult, error) { - results := &models.ProjectQueryResult{ - Total: 2, - Projects: make([]*models.Project, 0), - } - - results.Projects = append(results.Projects, &models.Project{ProjectID: 0, Name: "repo1"}) - results.Projects = append(results.Projects, &models.Project{ProjectID: 1, Name: "repo2"}) - - return results, nil -} - -func (mpm *mockProjectManager) IsPublic(projectIDOrName interface{}) (bool, error) { - return false, errors.New("Not implemented") -} - -func (mpm *mockProjectManager) Exists(projectIDOrName interface{}) (bool, error) { - return false, errors.New("Not implemented") -} - -// get all public project -func (mpm *mockProjectManager) GetPublic() ([]*models.Project, error) { - return nil, errors.New("Not implemented") -} - -// if the project manager uses a metadata manager, return it, otherwise return nil -func (mpm *mockProjectManager) GetMetadataManager() metamgr.ProjectMetadataManager { - return nil -} - -// Utility methods -// Start the mock frontend and backend servers -func startMockServers() error { - mockServer.Start() - backendURL, err := url.Parse(getTheAddrOfMockServer()) - if err != nil { - return err - } - - mockController, err = NewController(backendURL) - if err != nil { - return err - } - - frontServer.Start() - return nil -} - -// Stop the mock frontend and backend servers -func stopMockServers() { - frontServer.Close() - mockServer.Close() -} - -// Get the address of the backend mock server -func getTheAddrOfMockServer() string { - return mockServer.URL -} - -// Get the address of the frontend server -func getTheAddrOfFrontServer() string { - return frontServer.URL -} - // Mock content +var chartVersionsOfHarbor = ` +[ + { + "name": "harbor", + "home": "https://github.com/vmware/harbor", + "sources": [ + "https://github.com/vmware/harbor/tree/master/contrib/helm/harbor" + ], + "version": "0.2.1", + "description": "An Enterprise-class Docker Registry by VMware", + "keywords": [ + "vmware", + "docker", + "registry", + "harbor" + ], + "maintainers": [ + { + "name": "Jesse Hu", + "email": "huh@vmware.com" + }, + { + "name": "paulczar", + "email": "username.taken@gmail.com" + } + ], + "engine": "gotpl", + "icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png", + "appVersion": "1.5.0", + "urls": [ + "charts/harbor-0.2.1.tgz" + ], + "created": "2018-08-29T10:26:29.625749155Z", + "digest": "2538edf4ddb797af8e025f3bd6226270440110bbdb689bad48656a519a154236" + }, + { + "name": "harbor", + "home": "https://github.com/vmware/harbor", + "sources": [ + "https://github.com/vmware/harbor/tree/master/contrib/helm/harbor" + ], + "version": "0.2.0", + "description": "An Enterprise-class Docker Registry by VMware", + "keywords": [ + "vmware", + "docker", + "registry", + "harbor" + ], + "maintainers": [ + { + "name": "Jesse Hu", + "email": "huh@vmware.com" + }, + { + "name": "paulczar", + "email": "username.taken@gmail.com" + } + ], + "engine": "gotpl", + "icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png", + "appVersion": "1.5.0", + "urls": [ + "charts/harbor-0.2.0.tgz" + ], + "created": "2018-08-29T10:26:21.141611102Z", + "digest": "fc8aae8dade9f0dfca12e9f1085081c49843d30a063a3fa7eb42497e3ceb277c" + } +] +` var harborChartV = ` { "name": "harbor", diff --git a/src/chartserver/chart_operator.go b/src/chartserver/chart_operator.go index 563011359..2796233ce 100644 --- a/src/chartserver/chart_operator.go +++ b/src/chartserver/chart_operator.go @@ -11,6 +11,7 @@ import ( "github.com/Masterminds/semver" + "github.com/goharbor/harbor/src/common/models" hlog "github.com/goharbor/harbor/src/common/utils/log" "k8s.io/helm/pkg/chartutil" helm_repo "k8s.io/helm/pkg/repo" @@ -21,6 +22,15 @@ const ( valuesFileName = "values.yaml" ) +// ChartVersion extends the helm ChartVersion with additional labels +type ChartVersion struct { + helm_repo.ChartVersion + Labels []*models.Label `json:"labels"` +} + +// ChartVersions is an array of extended ChartVersion +type ChartVersions []*ChartVersion + // ChartVersionDetails keeps the detailed data info of the chart version type ChartVersionDetails struct { Metadata *helm_repo.ChartVersion `json:"metadata"` @@ -28,6 +38,7 @@ type ChartVersionDetails struct { Values map[string]interface{} `json:"values"` Files map[string]string `json:"files"` Security *SecurityReport `json:"security"` + Labels []*models.Label `json:"labels"` } // SecurityReport keeps the info related with security @@ -154,12 +165,12 @@ func (cho *ChartOperator) GetChartList(content []byte) ([]*ChartInfo, error) { } // GetChartVersions returns the chart versions -func (cho *ChartOperator) GetChartVersions(content []byte) (helm_repo.ChartVersions, error) { +func (cho *ChartOperator) GetChartVersions(content []byte) (ChartVersions, error) { if content == nil || len(content) == 0 { return nil, errors.New("zero content") } - chartVersions := make(helm_repo.ChartVersions, 0) + chartVersions := make(ChartVersions, 0) if err := json.Unmarshal(content, &chartVersions); err != nil { return nil, err } diff --git a/src/chartserver/controller.go b/src/chartserver/controller.go index 6603047aa..f5ec12614 100644 --- a/src/chartserver/controller.go +++ b/src/chartserver/controller.go @@ -5,7 +5,6 @@ import ( "fmt" "net/url" "os" - "strings" hlog "github.com/goharbor/harbor/src/common/utils/log" ) @@ -25,20 +24,21 @@ type Credential struct { // A reverse proxy will be created and managed to proxy the related traffics between API and // backend chart server type Controller struct { + // Proxy used to to transfer the traffic of requests + // It's mainly used to talk to the backend chart server + trafficProxy *ProxyEngine + + // Parse and process the chart version to provide required info data + chartOperator *ChartOperator + + // HTTP client used to call the realted APIs of the backend chart repositories + apiClient *ChartClient + // The access endpoint of the backend chart repository server - backendServerAddr *url.URL + backendServerAddress *url.URL - // To cover the server info and status requests - baseHandler *BaseHandler - - // To cover the chart repository requests - repositoryHandler *RepositoryHandler - - // To cover all the manipulation requests - manipulationHandler *ManipulationHandler - - // To cover the other utility requests - utilityHandler *UtilityHandler + // Cache the chart data + chartCache *ChartCache } // NewController is constructor of the chartserver.Controller @@ -53,15 +53,6 @@ func NewController(backendServer *url.URL) (*Controller, error) { Password: os.Getenv(passwordKey), } - // Use customized reverse proxy - proxy := NewProxyEngine(backendServer, cred) - - // Create http client with customized timeouts - client := NewChartClient(cred) - - // Initialize chart operator for use - operator := &ChartOperator{} - // Creat cache cacheCfg, err := getCacheConfig() if err != nil { @@ -75,79 +66,18 @@ func NewController(backendServer *url.URL) (*Controller, error) { } return &Controller{ - backendServerAddr: backendServer, - baseHandler: &BaseHandler{proxy}, - repositoryHandler: &RepositoryHandler{ - trafficProxy: proxy, - apiClient: client, - backendServerAddress: backendServer, - }, - manipulationHandler: &ManipulationHandler{ - trafficProxy: proxy, - chartOperator: operator, - apiClient: client, - backendServerAddress: backendServer, - chartCache: cache, - }, - utilityHandler: &UtilityHandler{ - apiClient: client, - backendServerAddress: backendServer, - chartOperator: operator, - }, + backendServerAddress: backendServer, + // Use customized reverse proxy + trafficProxy: NewProxyEngine(backendServer, cred), + // Initialize chart operator for use + chartOperator: &ChartOperator{}, + // Create http client with customized timeouts + apiClient: NewChartClient(cred), + chartCache: cache, }, nil } -// GetBaseHandler returns the reference of BaseHandler -func (c *Controller) GetBaseHandler() *BaseHandler { - return c.baseHandler -} - -// GetRepositoryHandler returns the reference of RepositoryHandler -func (c *Controller) GetRepositoryHandler() *RepositoryHandler { - return c.repositoryHandler -} - -// GetManipulationHandler returns the reference of ManipulationHandler -func (c *Controller) GetManipulationHandler() *ManipulationHandler { - return c.manipulationHandler -} - -// GetUtilityHandler returns the reference of UtilityHandler -func (c *Controller) GetUtilityHandler() *UtilityHandler { - return c.utilityHandler -} - -// What's the cache driver if it is set -func parseCacheDriver() (string, bool) { - driver, ok := os.LookupEnv(cacheDriverENVKey) - return strings.ToLower(driver), ok -} - -// Get and parse the configuration for the chart cache -func getCacheConfig() (*ChartCacheConfig, error) { - driver, isSet := parseCacheDriver() - if !isSet { - return nil, nil - } - - if driver != cacheDriverMem && driver != cacheDriverRedis { - return nil, fmt.Errorf("cache driver '%s' is not supported, only support 'memory' and 'redis'", driver) - } - - if driver == cacheDriverMem { - return &ChartCacheConfig{ - DriverType: driver, - }, nil - } - - redisConfigV := os.Getenv(redisENVKey) - redisCfg, err := parseRedisConfig(redisConfigV) - if err != nil { - return nil, fmt.Errorf("failed to parse redis configurations from '%s' with error: %s", redisCfg, err) - } - - return &ChartCacheConfig{ - DriverType: driver, - Config: redisCfg, - }, nil +// APIPrefix returns the API prefix path of calling backend chart service. +func (c *Controller) APIPrefix(namespace string) string { + return fmt.Sprintf("%s/api/%s/charts", c.backendServerAddress.String(), namespace) } diff --git a/src/chartserver/controller_test.go b/src/chartserver/controller_test.go index 0bbc52228..31837db59 100644 --- a/src/chartserver/controller_test.go +++ b/src/chartserver/controller_test.go @@ -1,225 +1,21 @@ package chartserver import ( - "encoding/json" "fmt" - "io/ioutil" - "net/http" - "strings" "testing" - - "github.com/ghodss/yaml" - helm_repo "k8s.io/helm/pkg/repo" ) -// Prepare, start the mock servers -func TestStartServers(t *testing.T) { - if err := startMockServers(); err != nil { - t.Fatal(err) - } -} - -// Test /health -func TestGetHealthOfBaseHandler(t *testing.T) { - content, err := httpClient.GetContent(fmt.Sprintf("%s/health", getTheAddrOfFrontServer())) +// Test controller +func TestController(t *testing.T) { + s, c, err := createMockObjects() if err != nil { t.Fatal(err) } + defer s.Close() - status := make(map[string]interface{}) - if err := json.Unmarshal(content, &status); err != nil { - t.Fatalf("Unmarshal error: %s, %s", err, content) - } - healthy, ok := status["health"].(bool) - if !ok || !healthy { - t.Fatalf("Expect healthy of server to be 'true' but got %v", status["health"]) + prefix := c.APIPrefix("fake") + expected := fmt.Sprintf("%s/api/%s/charts", s.URL, "fake") + if prefix != expected { + t.Fatalf("expect '%s' but got '%s'", expected, prefix) } } - -// Get /repo1/index.yaml -func TestGetIndexYamlByRepo(t *testing.T) { - indexFile, err := getIndexYaml("/repo1/index.yaml") - if err != nil { - t.Fatal(err) - } - - if len(indexFile.Entries) != 3 { - t.Fatalf("Expect index file with 3 entries, but got %d", len(indexFile.Entries)) - } -} - -// Test get /index.yaml -func TestGetUnifiedYamlFile(t *testing.T) { - indexFile, err := getIndexYaml("/index.yaml") - if err != nil { - t.Fatal(err) - } - - if len(indexFile.Entries) != 5 { - t.Fatalf("Expect index file with 5 entries, but got %d", len(indexFile.Entries)) - } - - _, ok := indexFile.Entries["repo1/harbor"] - if !ok { - t.Fatal("Expect chart entry 'repo1/harbor' but got nothing") - } - - _, ok = indexFile.Entries["repo2/harbor"] - if !ok { - t.Fatal("Expect chart entry 'repo2/harbor' but got nothing") - } -} - -// Test download /:repo/charts/chart.tar -// Use this case to test the proxy function -func TestDownloadChart(t *testing.T) { - content, err := httpClient.GetContent(fmt.Sprintf("%s/repo1/charts/harbor-0.2.0.tgz", getTheAddrOfFrontServer())) - if err != nil { - t.Fatal(err) - } - - gotSize := len(content) - expectSize := len(helmChartContent) - - if gotSize != expectSize { - t.Fatalf("Expect %d bytes data but got %d bytes", expectSize, gotSize) - } -} - -// Test get /api/:repo/charts -func TestRetrieveChartList(t *testing.T) { - content, err := httpClient.GetContent(fmt.Sprintf("%s/api/repo1/charts", getTheAddrOfFrontServer())) - if err != nil { - t.Fatal(err) - } - - chartList := make([]*ChartInfo, 0) - err = json.Unmarshal(content, &chartList) - if err != nil { - t.Fatalf("Unmarshal error: %s", err) - } - - if len(chartList) != 2 { - t.Fatalf("Expect to get 2 charts in the list but got %d", len(chartList)) - } - - foundItem := false - for _, chartInfo := range chartList { - if chartInfo.Name == "hello-helm" && chartInfo.TotalVersions == 2 { - foundItem = true - break - } - } - - if !foundItem { - t.Fatalf("Expect chart 'hello-helm' with 2 versions but got nothing") - } -} - -// Test get /api/:repo/charts/:chart_name/:version -func TestGetChartVersion(t *testing.T) { - content, err := httpClient.GetContent(fmt.Sprintf("%s/api/repo1/charts/harbor/0.2.0", getTheAddrOfFrontServer())) - if err != nil { - t.Fatal(err) - } - - chartVersion := &ChartVersionDetails{} - if err = json.Unmarshal(content, chartVersion); err != nil { - t.Fatalf("Unmarshal error: %s", err) - } - - if chartVersion.Metadata.Name != "harbor" { - t.Fatalf("Expect harbor chart version but got %s", chartVersion.Metadata.Name) - } - - if chartVersion.Metadata.Version != "0.2.0" { - t.Fatalf("Expect version '0.2.0' but got version %s", chartVersion.Metadata.Version) - } - - if len(chartVersion.Dependencies) != 1 { - t.Fatalf("Expect 1 dependency but got %d ones", len(chartVersion.Dependencies)) - } - - if len(chartVersion.Values) != 99 { - t.Fatalf("Expect 99 k-v values but got %d", len(chartVersion.Values)) - } -} - -// Test get /api/:repo/charts/:chart_name/:version with none-existing version -func TestGetChartVersionWithError(t *testing.T) { - _, err := httpClient.GetContent(fmt.Sprintf("%s/api/repo1/charts/harbor/1.0.0", getTheAddrOfFrontServer())) - if err == nil { - t.Fatal("Expect an error but got nil") - } -} - -// Get /api/repo1/charts/harbor -// 401 will be rewritten to 500 with specified error -func TestResponseRewrite(t *testing.T) { - response, err := http.Get(fmt.Sprintf("%s/api/repo1/charts/harbor", getTheAddrOfFrontServer())) - if err != nil { - t.Fatal(err) - } - - if response.StatusCode != http.StatusInternalServerError { - t.Fatalf("Expect status code 500 but got %d", response.StatusCode) - } - - bytes, err := ioutil.ReadAll(response.Body) - if err != nil { - t.Fatalf("Read bytes from http response failed with error: %s", err) - } - defer response.Body.Close() - - errObj := make(map[string]interface{}) - if err = json.Unmarshal(bytes, &errObj); err != nil { - t.Fatalf("Unmarshal error: %s", err) - } - - if msg, ok := errObj["error"]; !ok { - t.Fatal("Expect an error message from server but got nothing") - } else { - if !strings.Contains(msg.(string), "operation request from unauthorized source is rejected") { - t.Fatal("Missing the required error message") - } - } -} - -// Test the chart searching -func TestChartSearching(t *testing.T) { - namespaces := []string{"repo1", "repo2"} - q := "harbor" - - results, err := mockController.GetRepositoryHandler().SearchChart(q, namespaces) - if err != nil { - t.Fatalf("expect nil error but got '%s'", err) - } - - if len(results) != 2 { - t.Fatalf("expect 2 results but got %d", len(results)) - } -} - -// Clear environments -func TestStopServers(t *testing.T) { - stopMockServers() -} - -// Utility method for getting index yaml file -func getIndexYaml(path string) (*helm_repo.IndexFile, error) { - content, err := httpClient.GetContent(fmt.Sprintf("%s%s", getTheAddrOfFrontServer(), path)) - if err != nil { - return nil, err - } - - indexFile := &helm_repo.IndexFile{} - if err := yaml.Unmarshal(content, indexFile); err != nil { - return nil, fmt.Errorf("Unmarshal error: %s", err) - } - - if indexFile == nil { - return nil, fmt.Errorf("Got nil index yaml file") - } - - return indexFile, nil -} diff --git a/src/chartserver/handler_interface.go b/src/chartserver/handler_interface.go new file mode 100644 index 000000000..602a68467 --- /dev/null +++ b/src/chartserver/handler_interface.go @@ -0,0 +1,89 @@ +package chartserver + +import ( + "net/http" + + "k8s.io/helm/cmd/helm/search" + helm_repo "k8s.io/helm/pkg/repo" +) + +// ServiceHandler defines the related methods to handle kinds of chart service requests. +type ServiceHandler interface { + // ListCharts lists all the charts under the specified namespace. + // + // namespace string: the chart namespace. + // + // If succeed, a chart info list with nil error will be returned; + // otherwise, a non-nil error will be got. + ListCharts(namespace string) ([]*ChartInfo, error) + + // Get all the chart versions of the specified chart under the namespace. + // + // namespace string: the chart namespace. + // chartName string: the name of the chart, e.g: "harbor" + // + // If succeed, a chart version list with nil error will be returned; + // otherwise, a non-nil error will be got. + GetChart(namespace, chartName string) (helm_repo.ChartVersions, error) + + // Get the detailed info of the specified chart version under the namespace. + // The detailed info includes chart summary, dependencies, values and signature status etc. + // + // namespace string: the chart namespace. + // chartName string: the name of the chart, e.g: "harbor" + // version string: the SemVer version of the chart, e.g: "0.2.0" + // + // If succeed, chart version details with nil error will be returned; + // otherwise, a non-nil error will be got. + GetChartVersionDetails(namespace, chartName, version string) (*ChartVersionDetails, error) + + // SearchChart search charts in the specified namespaces with the keyword q. + // RegExp mode is enabled as default. + // For each chart, only the latest version will shown in the result list if matched to avoid duplicated entries. + // Keep consistent with `helm search` command. + // + // q string : the searching keyword + // namespaces []string : the search namespace scope + // + // If succeed, a search result list with nil error will be returned; + // otherwise, a non-nil error will be got. + SearchChart(q string, namespaces []string) ([]*search.Result, error) + + // GetIndexFile will read the index.yaml under all namespaces and merge them as a single one + // Please be aware that, to support this function, the backend chart repository server should + // enable multi-tenancies + // + // namespaces []string : all the namespaces with accessing permissions + // + // If succeed, a unified merged index file with nil error will be returned; + // otherwise, a non-nil error will be got. + GetIndexFile(namespaces []string) (*helm_repo.IndexFile, error) + + // Get the chart summary of the specified chart version. + // + // namespace string: the chart namespace. + // chartName string: the name of the chart, e.g: "harbor" + // version string: the SemVer version of the chart, e.g: "0.2.0" + // + // If succeed, chart version summary with nil error will be returned; + // otherwise, a non-nil error will be got. + GetChartVersion(namespace, name, version string) (*helm_repo.ChartVersion, error) + + // DeleteChart deletes all the chart versions of the specified chart under the namespace. + // + // namespace string: the chart namespace. + // chartName string: the name of the chart, e.g: "harbor" + // + // If succeed, a nil error will be returned; + // otherwise, a non-nil error will be got. + DeleteChart(namespace, chartName string) error +} + +// ProxyTrafficHandler defines the handler methods to handle the proxy traffic. +type ProxyTrafficHandler interface { + // Proxy the traffic to the backended server + // + // Req *http.Request : The incoming http request + // w http.ResponseWriter : The response writer reference + ProxyTraffic(w http.ResponseWriter, req *http.Request) +} diff --git a/src/chartserver/handler_manipulation.go b/src/chartserver/handler_manipulation.go new file mode 100644 index 000000000..6957fa14a --- /dev/null +++ b/src/chartserver/handler_manipulation.go @@ -0,0 +1,93 @@ +package chartserver + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/ghodss/yaml" + helm_repo "k8s.io/helm/pkg/repo" +) + +// ListCharts gets the chart list under the namespace +// See @ServiceHandler.ListCharts +func (c *Controller) ListCharts(namespace string) ([]*ChartInfo, error) { + if len(strings.TrimSpace(namespace)) == 0 { + return nil, errors.New("empty namespace when getting chart list") + } + + content, err := c.apiClient.GetContent(c.APIPrefix(namespace)) + if err != nil { + return nil, err + } + + return c.chartOperator.GetChartList(content) +} + +// GetChart returns all the chart versions under the specified chart +// See @ServiceHandler.GetChart +func (c *Controller) GetChart(namespace, chartName string) (ChartVersions, error) { + if len(namespace) == 0 { + return nil, errors.New("empty name when getting chart versions") + } + + if len(chartName) == 0 { + return nil, errors.New("no chart name specified") + } + + url := fmt.Sprintf("%s/%s", c.APIPrefix(namespace), chartName) + data, err := c.apiClient.GetContent(url) + if err != nil { + return nil, err + } + + versions := make(ChartVersions, 0) + if err := json.Unmarshal(data, &versions); err != nil { + return nil, err + } + + return versions, nil +} + +// DeleteChartVersion will delete the specified version of the chart +// See @ServiceHandler.DeleteChartVersion +func (c *Controller) DeleteChartVersion(namespace, chartName, version string) error { + if len(namespace) == 0 { + return errors.New("empty namespace when deleting chart version") + } + + if len(chartName) == 0 || len(version) == 0 { + return errors.New("invalid chart for deleting") + } + + url := fmt.Sprintf("%s/%s/%s", c.APIPrefix(namespace), chartName, version) + + return c.apiClient.DeleteContent(url) +} + +// GetChartVersion returns the summary of the specified chart version. +// See @ServiceHandler.GetChartVersion +func (c *Controller) GetChartVersion(namespace, name, version string) (*helm_repo.ChartVersion, error) { + if len(namespace) == 0 { + return nil, errors.New("empty namespace when getting summary of chart version") + } + + if len(name) == 0 || len(version) == 0 { + return nil, errors.New("invalid chart when getting summary") + } + + url := fmt.Sprintf("%s/%s/%s", c.APIPrefix(namespace), name, version) + + content, err := c.apiClient.GetContent(url) + if err != nil { + return nil, err + } + + chartVersion := &helm_repo.ChartVersion{} + if err := yaml.Unmarshal(content, chartVersion); err != nil { + return nil, err + } + + return chartVersion, nil +} diff --git a/src/chartserver/handler_manipulation_test.go b/src/chartserver/handler_manipulation_test.go new file mode 100644 index 000000000..602d71e2a --- /dev/null +++ b/src/chartserver/handler_manipulation_test.go @@ -0,0 +1,88 @@ +package chartserver + +import ( + "testing" +) + +// Test get /api/:repo/charts/harbor +func TestGetChart(t *testing.T) { + s, c, err := createMockObjects() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + versions, err := c.GetChart("repo1", "harbor") + if err != nil { + t.Fatal(err) + } + + if len(versions) != 2 { + t.Fatalf("expect 2 chart versions of harbor but got %d", len(versions)) + } +} + +// Test delete /api/:repo/charts/harbor/0.2.0 +func TestDeleteChartVersion(t *testing.T) { + s, c, err := createMockObjects() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + if err := c.DeleteChartVersion("repo1", "harbor", "0.2.0"); err != nil { + t.Fatal(err) + } +} + +// Test get /api/:repo/charts +func TestRetrieveChartList(t *testing.T) { + s, c, err := createMockObjects() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + chartList, err := c.ListCharts("repo1") + if err != nil { + t.Fatal(err) + } + + if len(chartList) != 2 { + t.Fatalf("Expect to get 2 charts in the list but got %d", len(chartList)) + } + + foundItem := false + for _, chartInfo := range chartList { + if chartInfo.Name == "hello-helm" && chartInfo.TotalVersions == 2 { + foundItem = true + break + } + } + + if !foundItem { + t.Fatalf("Expect chart 'hello-helm' with 2 versions but got nothing") + } +} + +// Test the GetChartVersion in utility handler +func TestGetChartVersionSummary(t *testing.T) { + s, c, err := createMockObjects() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + chartV, err := c.GetChartVersion("repo1", "harbor", "0.2.0") + if err != nil { + t.Fatal(err) + } + + if chartV.GetName() != "harbor" { + t.Fatalf("expect chart name 'harbor' but got '%s'", chartV.GetName()) + } + + if chartV.GetVersion() != "0.2.0" { + t.Fatalf("expect chart version '0.2.0' but got '%s'", chartV.GetVersion()) + } +} diff --git a/src/chartserver/handler_proxy_traffic.go b/src/chartserver/handler_proxy_traffic.go new file mode 100644 index 000000000..72b9147fc --- /dev/null +++ b/src/chartserver/handler_proxy_traffic.go @@ -0,0 +1,12 @@ +package chartserver + +import ( + "net/http" +) + +// ProxyTraffic implements the interface method. +func (c *Controller) ProxyTraffic(w http.ResponseWriter, req *http.Request) { + if c.trafficProxy != nil { + c.trafficProxy.ServeHTTP(w, req) + } +} diff --git a/src/chartserver/handler_proxy_traffic_test.go b/src/chartserver/handler_proxy_traffic_test.go new file mode 100644 index 000000000..261e49636 --- /dev/null +++ b/src/chartserver/handler_proxy_traffic_test.go @@ -0,0 +1,136 @@ +package chartserver + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/ghodss/yaml" + helm_repo "k8s.io/helm/pkg/repo" +) + +// The frontend server +var frontServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mockController.ProxyTraffic(w, r) +})) + +var mockServer *httptest.Server +var mockController *Controller + +// Prepare case +func TestStartMockServers(t *testing.T) { + s, c, err := createMockObjects() + if err != nil { + t.Fatal(err) + } + mockController = c + mockServer = s + + frontServer.Start() +} + +// Test /health +func TestGetHealthOfBaseHandler(t *testing.T) { + content, err := httpClient.GetContent(fmt.Sprintf("%s/api/chartrepo/health", frontServer.URL)) + if err != nil { + t.Fatal(err) + } + + status := make(map[string]interface{}) + if err := json.Unmarshal(content, &status); err != nil { + t.Fatalf("Unmarshal error: %s, %s", err, content) + } + healthy, ok := status["health"].(bool) + if !ok || !healthy { + t.Fatalf("Expect healthy of server to be 'true' but got %v", status["health"]) + } +} + +// Get /repo1/index.yaml +func TestGetIndexYamlByRepo(t *testing.T) { + indexFile, err := getIndexYaml("/chartrepo/repo1/index.yaml") + if err != nil { + t.Fatal(err) + } + + if len(indexFile.Entries) != 3 { + t.Fatalf("Expect index file with 3 entries, but got %d", len(indexFile.Entries)) + } +} + +// Test download /:repo/charts/chart.tar +// Use this case to test the proxy function +func TestDownloadChart(t *testing.T) { + content, err := httpClient.GetContent(fmt.Sprintf("%s/chartrepo/repo1/charts/harbor-0.2.0.tgz", frontServer.URL)) + if err != nil { + t.Fatal(err) + } + + gotSize := len(content) + expectSize := len(helmChartContent) + + if gotSize != expectSize { + t.Fatalf("Expect %d bytes data but got %d bytes", expectSize, gotSize) + } +} + +// Get /api/repo1/charts/harbor +// 401 will be rewritten to 500 with specified error +func TestResponseRewrite(t *testing.T) { + response, err := http.Get(fmt.Sprintf("%s/chartrepo/repo3/charts/harbor-0.8.1.tgz", frontServer.URL)) + if err != nil { + t.Fatal(err) + } + + if response.StatusCode != http.StatusInternalServerError { + t.Fatalf("Expect status code 500 but got %d", response.StatusCode) + } + + bytes, err := ioutil.ReadAll(response.Body) + if err != nil { + t.Fatalf("Read bytes from http response failed with error: %s", err) + } + defer response.Body.Close() + + errObj := make(map[string]interface{}) + if err = json.Unmarshal(bytes, &errObj); err != nil { + t.Fatalf("Unmarshal error: %s", err) + } + + if msg, ok := errObj["error"]; !ok { + t.Fatal("Expect an error message from server but got nothing") + } else { + if !strings.Contains(msg.(string), "operation request from unauthorized source is rejected") { + t.Fatal("Missing the required error message") + } + } +} + +// Clear env +func TestStopMockServers(t *testing.T) { + frontServer.Close() + mockServer.Close() +} + +// Utility method for getting index yaml file +func getIndexYaml(path string) (*helm_repo.IndexFile, error) { + content, err := httpClient.GetContent(fmt.Sprintf("%s%s", frontServer.URL, path)) + if err != nil { + return nil, err + } + + indexFile := &helm_repo.IndexFile{} + if err := yaml.Unmarshal(content, indexFile); err != nil { + return nil, fmt.Errorf("Unmarshal error: %s", err) + } + + if indexFile == nil { + return nil, fmt.Errorf("Got nil index yaml file") + } + + return indexFile, nil +} diff --git a/src/chartserver/repo_handler.go b/src/chartserver/handler_repo.go similarity index 55% rename from src/chartserver/repo_handler.go rename to src/chartserver/handler_repo.go index 31330bc9e..db3fe0c21 100644 --- a/src/chartserver/repo_handler.go +++ b/src/chartserver/handler_repo.go @@ -1,17 +1,12 @@ package chartserver import ( - "encoding/json" "fmt" - "net/http" - "net/url" "path" "sync" "time" "github.com/ghodss/yaml" - "github.com/goharbor/harbor/src/ui/filter" - "k8s.io/helm/cmd/helm/search" helm_repo "k8s.io/helm/pkg/repo" hlog "github.com/goharbor/harbor/src/common/utils/log" @@ -24,124 +19,33 @@ const ( searchMaxScore = 25 ) -// RepositoryHandler defines all the handlers to handle the requests related with chart repository -// e.g: index.yaml and downloading chart objects -type RepositoryHandler struct { - // Proxy used to to transfer the traffic of requests - // It's mainly used to talk to the backend chart server - trafficProxy *ProxyEngine - - // HTTP client used to call the realted APIs of the backend chart repositories - apiClient *ChartClient - - // Point to the url of the backend server - backendServerAddress *url.URL -} - // Result returned by worker type processedResult struct { namespace string indexFileOfRepo *helm_repo.IndexFile } -// GetIndexFileWithNS will read the index.yaml data under the specified namespace -func (rh *RepositoryHandler) GetIndexFileWithNS(w http.ResponseWriter, req *http.Request) { - rh.trafficProxy.ServeHTTP(w, req) -} - // GetIndexFile will read the index.yaml under all namespaces and merge them as a single one // Please be aware that, to support this function, the backend chart repository server should // enable multi-tenancies -func (rh *RepositoryHandler) GetIndexFile(w http.ResponseWriter, req *http.Request) { - // Get project manager references - projectMgr, err := filter.GetProjectManager(req) - if err != nil { - WriteInternalError(w, err) - return +// +// See @ServiceHandler.GetIndexFile +func (c *Controller) GetIndexFile(namespaces []string) (*helm_repo.IndexFile, error) { + if namespaces == nil || len(namespaces) == 0 { + return emptyIndexFile(), nil } - // Get all the projects - results, err := projectMgr.List(nil) - if err != nil { - WriteInternalError(w, err) - return - } - - // If no projects existing, return empty index.yaml content immediately - if results.Total == 0 { - w.Write(emptyIndexFile()) - return - } - - namespaces := []string{} - for _, p := range results.Projects { - namespaces = append(namespaces, p.Name) - } - - mergedIndexFile, err := rh.getIndexYaml(namespaces) - if err != nil { - WriteInternalError(w, err) - return - } - - bytes, err := yaml.Marshal(mergedIndexFile) - if err != nil { - WriteInternalError(w, err) - return - } - - w.Write(bytes) -} - -// DownloadChartObject will download the stored chart object to the client -// e.g: helm install -func (rh *RepositoryHandler) DownloadChartObject(w http.ResponseWriter, req *http.Request) { - rh.trafficProxy.ServeHTTP(w, req) -} - -// SearchChart search charts in the specified namespaces with the keyword q. -// RegExp mode is enabled as default. -// For each chart, only the latest version will shown in the result list if matched to avoid duplicated entries. -// Keep consistent with `helm search` command. -func (rh *RepositoryHandler) SearchChart(q string, namespaces []string) ([]*search.Result, error) { - if len(q) == 0 || len(namespaces) == 0 { - // Return empty list - return []*search.Result{}, nil - } - - // Get the merged index yaml file of the namespaces - ind, err := rh.getIndexYaml(namespaces) - if err != nil { - return nil, err - } - - // Build the search index - index := search.NewIndex() - // As the repo name is already merged into the index yaml, we use empty repo name. - // Set 'All' to false to return only one version for each chart. - index.AddRepo("", ind, false) - - // Search - // RegExp is enabled - results, err := index.Search(q, searchMaxScore, true) - if err != nil { - return nil, err - } - - // Sort results. - search.SortScore(results) - - return results, nil + return c.getIndexYaml(namespaces) } // getIndexYaml will get the index yaml files for all the namespaces and merge them // as one unified index yaml file. -func (rh *RepositoryHandler) getIndexYaml(namespaces []string) (*helm_repo.IndexFile, error) { +func (c *Controller) getIndexYaml(namespaces []string) (*helm_repo.IndexFile, error) { // The final merged index file mergedIndexFile := &helm_repo.IndexFile{ APIVersion: "v1", Entries: make(map[string]helm_repo.ChartVersions), - Generated: time.Now(), + Generated: time.Now().Round(time.Second), PublicKeys: []string{}, } @@ -175,7 +79,7 @@ func (rh *RepositoryHandler) getIndexYaml(namespaces []string) (*helm_repo.Index }() for res := range resultChan { - rh.mergeIndexFile(res.namespace, mergedIndexFile, res.indexFileOfRepo) + c.mergeIndexFile(res.namespace, mergedIndexFile, res.indexFileOfRepo) } }() @@ -203,7 +107,7 @@ LOOP: workerPool <- struct{}{} }() - indexFile, err := rh.getIndexYamlWithNS(ns) + indexFile, err := c.getIndexYamlWithNS(ns) if err != nil { if len(errorChan) == 0 { //Only need one error as failure signal @@ -254,13 +158,13 @@ LOOP: } // Get the index yaml file under the specified namespace from the backend server -func (rh *RepositoryHandler) getIndexYamlWithNS(namespace string) (*helm_repo.IndexFile, error) { +func (c *Controller) getIndexYamlWithNS(namespace string) (*helm_repo.IndexFile, error) { // Join url path url := path.Join(namespace, "index.yaml") - url = fmt.Sprintf("%s/%s", rh.backendServerAddress.String(), url) + url = fmt.Sprintf("%s/%s", c.backendServerAddress.String(), url) hlog.Debugf("Getting index.yaml from '%s'", url) - content, err := rh.apiClient.GetContent(url) + content, err := c.apiClient.GetContent(url) if err != nil { return nil, err } @@ -276,7 +180,7 @@ func (rh *RepositoryHandler) getIndexYamlWithNS(namespace string) (*helm_repo.In // Merge the content of mergingIndexFile to the baseIndex // The chart url should be without --chart-url prefix -func (rh *RepositoryHandler) mergeIndexFile(namespace string, +func (c *Controller) mergeIndexFile(namespace string, baseIndex *helm_repo.IndexFile, mergingIndexFile *helm_repo.IndexFile) { // Append entries @@ -304,12 +208,9 @@ func (rh *RepositoryHandler) mergeIndexFile(namespace string, } // Generate empty index file -func emptyIndexFile() []byte { +func emptyIndexFile() *helm_repo.IndexFile { emptyIndexFile := &helm_repo.IndexFile{} - emptyIndexFile.Generated = time.Now() + emptyIndexFile.Generated = time.Now().Round(time.Second) - // Ignore the error - rawData, _ := json.Marshal(emptyIndexFile) - - return rawData + return emptyIndexFile } diff --git a/src/chartserver/handler_repo_test.go b/src/chartserver/handler_repo_test.go new file mode 100644 index 000000000..69f83f230 --- /dev/null +++ b/src/chartserver/handler_repo_test.go @@ -0,0 +1,32 @@ +package chartserver + +import "testing" + +// Test get /index.yaml +func TestGetIndexFile(t *testing.T) { + s, c, err := createMockObjects() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + namespaces := []string{"repo1", "repo2"} + indexFile, err := c.GetIndexFile(namespaces) + if err != nil { + t.Fatal(err) + } + + if len(indexFile.Entries) != 5 { + t.Fatalf("Expect index file with 5 entries, but got %d", len(indexFile.Entries)) + } + + _, ok := indexFile.Entries["repo1/harbor"] + if !ok { + t.Fatal("Expect chart entry 'repo1/harbor' but got nothing") + } + + _, ok = indexFile.Entries["repo2/harbor"] + if !ok { + t.Fatal("Expect chart entry 'repo2/harbor' but got nothing") + } +} diff --git a/src/chartserver/handler_utility.go b/src/chartserver/handler_utility.go new file mode 100644 index 000000000..1342f614a --- /dev/null +++ b/src/chartserver/handler_utility.go @@ -0,0 +1,208 @@ +package chartserver + +import ( + "errors" + "fmt" + "path" + "strings" + "sync" + + hlog "github.com/goharbor/harbor/src/common/utils/log" + "k8s.io/helm/cmd/helm/search" +) + +const ( + maxDeletionThreads = 10 +) + +// DeleteChart deletes all the chart versions of the specified chart under the namespace. +// See @ServiceHandler.DeleteChart +func (c *Controller) DeleteChart(namespace, chartName string) error { + if len(strings.TrimSpace(namespace)) == 0 { + return errors.New("empty namespace when deleting chart") + } + + if len(strings.TrimSpace(chartName)) == 0 { + return errors.New("empty chart name when deleting chart") + } + + url := fmt.Sprintf("%s/%s", c.APIPrefix(namespace), chartName) + + content, err := c.apiClient.GetContent(url) + if err != nil { + return err + } + + allVersions, err := c.chartOperator.GetChartVersions(content) + if err != nil { + return err + } + + // Let's delete the versions in parallel + // The number of goroutine is controlled by the const maxDeletionThreads + qSize := len(allVersions) + if qSize > maxDeletionThreads { + qSize = maxDeletionThreads + } + tokenQueue := make(chan struct{}, qSize) + errChan := make(chan error, 1) + waitGroup := new(sync.WaitGroup) + waitGroup.Add(len(allVersions)) + + // Append initial tokens + for i := 0; i < qSize; i++ { + tokenQueue <- struct{}{} + } + + // Collect errors + errs := make([]error, 0) + errWrapper := make(chan error, 1) + go func() { + defer func() { + // pass to the out func + if len(errs) > 0 { + errWrapper <- fmt.Errorf("%v", errs) + } + close(errWrapper) + }() + + for deletionErr := range errChan { + errs = append(errs, deletionErr) + } + }() + + // Schedule deletion tasks + for _, deletingVersion := range allVersions { + // Apply for token first + // If no available token, pending here + <-tokenQueue + + // Got one token + go func(deletingVersion *ChartVersion) { + defer func() { + // return the token back + tokenQueue <- struct{}{} + + // done + waitGroup.Done() + }() + + if err := c.DeleteChartVersion(namespace, chartName, deletingVersion.GetVersion()); err != nil { + errChan <- err + } + }(deletingVersion) + } + + // Wait all goroutines are done + waitGroup.Wait() + // Safe to quit error collection goroutine + close(errChan) + + err = <-errWrapper + + return err +} + +// GetChartVersionDetails get the specified version for one chart +// This handler should return the details of the chart version, +// maybe including metadata,dependencies and values etc. +// See @ServiceHandler.GetChartVersionDetails +func (c *Controller) GetChartVersionDetails(namespace, chartName, version string) (*ChartVersionDetails, error) { + chartV, err := c.GetChartVersion(namespace, chartName, version) + if err != nil { + return nil, err + } + + // Query cache + chartDetails := c.chartCache.GetChart(chartV.Digest) + if chartDetails == nil { + // NOT hit!! + content, err := c.getChartVersionContent(namespace, chartV.URLs[0]) + if err != nil { + return nil, err + } + + // Process bytes and get more details of chart version + chartDetails, err = c.chartOperator.GetChartDetails(content) + if err != nil { + return nil, err + } + chartDetails.Metadata = chartV + + // Put it into the cache for next access + c.chartCache.PutChart(chartDetails) + } else { + // Just logged + hlog.Debugf("Get detailed data from cache for chart: %s:%s (%s)", + chartDetails.Metadata.Name, + chartDetails.Metadata.Version, + chartDetails.Metadata.Digest) + } + // The change of prov file will not cause any influence to the digest of chart, + // and then the digital signature status should be not cached + // + // Generate the security report + // prov file share same endpoint with the chart version + // Just add .prov suffix to the chart version to form the path of prov file + // Anyway, there will be a report about the digital signature status + chartDetails.Security = &SecurityReport{ + Signature: &DigitalSignature{ + Signed: false, + }, + } + // Try to get the prov file to confirm if it is exitsing + provFilePath := fmt.Sprintf("%s.prov", chartV.URLs[0]) + provBytes, err := c.getChartVersionContent(namespace, provFilePath) + if err == nil && len(provBytes) > 0 { + chartDetails.Security.Signature.Signed = true + chartDetails.Security.Signature.Provenance = provFilePath + } else { + // Just log it + hlog.Errorf("Failed to get prov file for chart %s with error: %s, got %d bytes", chartV.Name, err.Error(), len(provBytes)) + } + + return chartDetails, nil +} + +// SearchChart search charts in the specified namespaces with the keyword q. +// RegExp mode is enabled as default. +// For each chart, only the latest version will shown in the result list if matched to avoid duplicated entries. +// Keep consistent with `helm search` command. +func (c *Controller) SearchChart(q string, namespaces []string) ([]*search.Result, error) { + if len(q) == 0 || len(namespaces) == 0 { + // Return empty list + return []*search.Result{}, nil + } + + // Get the merged index yaml file of the namespaces + ind, err := c.getIndexYaml(namespaces) + if err != nil { + return nil, err + } + + // Build the search index + index := search.NewIndex() + // As the repo name is already merged into the index yaml, we use empty repo name. + // Set 'All' to false to return only one version for each chart. + index.AddRepo("", ind, false) + + // Search + // RegExp is enabled + results, err := index.Search(q, searchMaxScore, true) + if err != nil { + return nil, err + } + + // Sort results. + search.SortScore(results) + + return results, nil +} + +// Get the content bytes of the chart version +func (c *Controller) getChartVersionContent(namespace string, subPath string) ([]byte, error) { + url := path.Join(namespace, subPath) + url = fmt.Sprintf("%s/%s", c.backendServerAddress.String(), url) + + return c.apiClient.GetContent(url) +} diff --git a/src/chartserver/handler_utility_test.go b/src/chartserver/handler_utility_test.go new file mode 100644 index 000000000..800ab3420 --- /dev/null +++ b/src/chartserver/handler_utility_test.go @@ -0,0 +1,83 @@ +package chartserver + +import ( + "testing" +) + +// Test the function DeleteChart +func TestDeleteChart(t *testing.T) { + s, c, err := createMockObjects() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + if err := c.DeleteChart("repo1", "harbor"); err != nil { + t.Fatal(err) + } +} + +// Test get /api/:repo/charts/:chart_name/:version +func TestGetChartVersion(t *testing.T) { + s, c, err := createMockObjects() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + chartVersion, err := c.GetChartVersionDetails("repo1", "harbor", "0.2.0") + if err != nil { + t.Fatal(err) + } + + if chartVersion.Metadata.Name != "harbor" { + t.Fatalf("Expect harbor chart version but got %s", chartVersion.Metadata.Name) + } + + if chartVersion.Metadata.Version != "0.2.0" { + t.Fatalf("Expect version '0.2.0' but got version %s", chartVersion.Metadata.Version) + } + + if len(chartVersion.Dependencies) != 1 { + t.Fatalf("Expect 1 dependency but got %d ones", len(chartVersion.Dependencies)) + } + + if len(chartVersion.Values) != 99 { + t.Fatalf("Expect 99 k-v values but got %d", len(chartVersion.Values)) + } +} + +// Test get /api/:repo/charts/:chart_name/:version with none-existing version +func TestGetChartVersionWithError(t *testing.T) { + s, c, err := createMockObjects() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + _, err = c.GetChartVersionDetails("repo1", "harbor", "1.0.0") + if err == nil { + t.Fatal("Expect an error but got nil") + } +} + +// Test the chart searching +func TestChartSearching(t *testing.T) { + s, c, err := createMockObjects() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + namespaces := []string{"repo1", "repo2"} + q := "harbor" + + results, err := c.SearchChart(q, namespaces) + if err != nil { + t.Fatalf("expect nil error but got '%s'", err) + } + + if len(results) != 2 { + t.Fatalf("expect 2 results but got %d", len(results)) + } +} diff --git a/src/chartserver/manipulation_handler.go b/src/chartserver/manipulation_handler.go deleted file mode 100644 index c21c9a449..000000000 --- a/src/chartserver/manipulation_handler.go +++ /dev/null @@ -1,199 +0,0 @@ -package chartserver - -import ( - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - "path" - "strings" - - "github.com/ghodss/yaml" - hlog "github.com/goharbor/harbor/src/common/utils/log" - helm_repo "k8s.io/helm/pkg/repo" -) - -const ( - // NamespaceContextKey is context key for the namespace - NamespaceContextKey ContextKey = ":repo" -) - -// ContextKey is defined for add value in the context of http request -type ContextKey string - -// ManipulationHandler includes all the handler methods for the purpose of manipulating the -// chart repository -type ManipulationHandler struct { - // Proxy used to to transfer the traffic of requests - // It's mainly used to talk to the backend chart server - trafficProxy *ProxyEngine - - // Parse and process the chart version to provide required info data - chartOperator *ChartOperator - - // HTTP client used to call the realted APIs of the backend chart repositories - apiClient *ChartClient - - // Point to the url of the backend server - backendServerAddress *url.URL - - // Cache the chart data - chartCache *ChartCache -} - -// ListCharts lists all the charts under the specified namespace -func (mh *ManipulationHandler) ListCharts(w http.ResponseWriter, req *http.Request) { - url := strings.TrimPrefix(req.URL.String(), "/") - url = fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), url) - - content, err := mh.apiClient.GetContent(url) - if err != nil { - WriteInternalError(w, err) - return - } - - chartList, err := mh.chartOperator.GetChartList(content) - if err != nil { - WriteInternalError(w, err) - return - } - - jsonData, err := json.Marshal(chartList) - if err != nil { - WriteInternalError(w, err) - return - } - - writeJSONData(w, jsonData) -} - -// GetChart returns all the chart versions under the specified chart -func (mh *ManipulationHandler) GetChart(w http.ResponseWriter, req *http.Request) { - mh.trafficProxy.ServeHTTP(w, req) -} - -// GetChartVersion get the specified version for one chart -// This handler should return the details of the chart version, -// maybe including metadata,dependencies and values etc. -func (mh *ManipulationHandler) GetChartVersion(w http.ResponseWriter, req *http.Request) { - chartV, err := mh.getChartVersion(req.URL.String()) - if err != nil { - WriteInternalError(w, err) - return - } - - // Get and check namespace - // even we get the data from cache - var namespace string - - repoValue := req.Context().Value(NamespaceContextKey) - if repoValue != nil { - if ns, ok := repoValue.(string); ok { - namespace = ns - } - } - - if len(strings.TrimSpace(namespace)) == 0 { - WriteInternalError(w, errors.New("failed to extract namespace from the request")) - return - } - - // Query cache - chartDetails := mh.chartCache.GetChart(chartV.Digest) - if chartDetails == nil { - // NOT hit!! - content, err := mh.getChartVersionContent(namespace, chartV.URLs[0]) - if err != nil { - WriteInternalError(w, err) - return - } - - // Process bytes and get more details of chart version - chartDetails, err = mh.chartOperator.GetChartDetails(content) - if err != nil { - WriteInternalError(w, err) - return - } - chartDetails.Metadata = chartV - - // Put it into the cache for next access - mh.chartCache.PutChart(chartDetails) - } else { - // Just logged - hlog.Debugf("Get detailed data from cache for chart: %s:%s (%s)", - chartDetails.Metadata.Name, - chartDetails.Metadata.Version, - chartDetails.Metadata.Digest) - } - // The change of prov file will not cause any influence to the digest of chart, - // and then the digital signature status should be not cached - // - // Generate the security report - // prov file share same endpoint with the chart version - // Just add .prov suffix to the chart version to form the path of prov file - // Anyway, there will be a report about the digital signature status - chartDetails.Security = &SecurityReport{ - Signature: &DigitalSignature{ - Signed: false, - }, - } - // Try to get the prov file to confirm if it is exitsing - provFilePath := fmt.Sprintf("%s.prov", chartV.URLs[0]) - provBytes, err := mh.getChartVersionContent(namespace, provFilePath) - if err == nil && len(provBytes) > 0 { - chartDetails.Security.Signature.Signed = true - chartDetails.Security.Signature.Provenance = provFilePath - } else { - // Just log it - hlog.Errorf("Failed to get prov file for chart %s with error: %s, got %d bytes", chartV.Name, err.Error(), len(provBytes)) - } - - bytes, err := json.Marshal(chartDetails) - if err != nil { - WriteInternalError(w, err) - return - } - - writeJSONData(w, bytes) -} - -// UploadChartVersion will save the new version of the chart to the backend storage -func (mh *ManipulationHandler) UploadChartVersion(w http.ResponseWriter, req *http.Request) { - mh.trafficProxy.ServeHTTP(w, req) -} - -// UploadProvenanceFile will save the provenance file of the chart to the backend storage -func (mh *ManipulationHandler) UploadProvenanceFile(w http.ResponseWriter, req *http.Request) { - mh.trafficProxy.ServeHTTP(w, req) -} - -// DeleteChartVersion will delete the specified version of the chart -func (mh *ManipulationHandler) DeleteChartVersion(w http.ResponseWriter, req *http.Request) { - mh.trafficProxy.ServeHTTP(w, req) -} - -// Get the basic metadata of chart version -func (mh *ManipulationHandler) getChartVersion(subPath string) (*helm_repo.ChartVersion, error) { - url := fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), strings.TrimPrefix(subPath, "/")) - - content, err := mh.apiClient.GetContent(url) - if err != nil { - return nil, err - } - - chartVersion := &helm_repo.ChartVersion{} - if err := yaml.Unmarshal(content, chartVersion); err != nil { - return nil, err - } - - return chartVersion, nil -} - -// Get the content bytes of the chart version -func (mh *ManipulationHandler) getChartVersionContent(namespace string, subPath string) ([]byte, error) { - url := path.Join(namespace, subPath) - url = fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), url) - - return mh.apiClient.GetContent(url) -} diff --git a/src/chartserver/reverse_proxy.go b/src/chartserver/reverse_proxy.go index e83ed8919..5d0c2479e 100644 --- a/src/chartserver/reverse_proxy.go +++ b/src/chartserver/reverse_proxy.go @@ -17,6 +17,11 @@ import ( const ( agentHarbor = "HARBOR" contentLengthHeader = "Content-Length" + + defaultRepo = "library" + rootUploadingEndpoint = "/api/chartrepo/charts" + rootIndexEndpoint = "/chartrepo/index.yaml" + chartRepoHealthEndpoint = "/api/chartrepo/health" ) // ProxyEngine is used to proxy the related traffics @@ -56,6 +61,7 @@ func director(target *url.URL, cred *Credential, req *http.Request) { // Overwrite the request URL to the target path req.URL.Scheme = target.Scheme req.URL.Host = target.Host + rewriteURLPath(req) req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path) if targetQuery == "" || req.URL.RawQuery == "" { req.URL.RawQuery = targetQuery + req.URL.RawQuery @@ -125,3 +131,34 @@ func singleJoiningSlash(a, b string) string { } return a + b } + +// Rewrite the incoming URL with the right backend URL pattern +// Remove 'chartrepo' from the endpoints of manipulation API +// Remove 'chartrepo' from the endpoints of repository services +func rewriteURLPath(req *http.Request) { + incomingURLPath := req.RequestURI + + // Health check endpoint + if incomingURLPath == chartRepoHealthEndpoint { + req.URL.Path = "/health" + return + } + + // Root uploading endpoint + if incomingURLPath == rootUploadingEndpoint { + req.URL.Path = strings.Replace(incomingURLPath, "chartrepo", defaultRepo, 1) + return + } + + // Repository endpoints + if strings.HasPrefix(incomingURLPath, "/chartrepo") { + req.URL.Path = strings.TrimPrefix(incomingURLPath, "/chartrepo") + return + } + + // API endpoints + if strings.HasPrefix(incomingURLPath, "/api/chartrepo") { + req.URL.Path = strings.Replace(incomingURLPath, "/chartrepo", "", 1) + return + } +} diff --git a/src/chartserver/utility_handler.go b/src/chartserver/utility_handler.go deleted file mode 100644 index e9aa9ccb6..000000000 --- a/src/chartserver/utility_handler.go +++ /dev/null @@ -1,163 +0,0 @@ -package chartserver - -import ( - "errors" - "fmt" - "net/url" - "strings" - "sync" - - "github.com/ghodss/yaml" - helm_repo "k8s.io/helm/pkg/repo" -) - -const ( - maxDeletionThreads = 10 -) - -// UtilityHandler provides utility methods -type UtilityHandler struct { - // Parse and process the chart version to provide required info data - chartOperator *ChartOperator - - // HTTP client used to call the realted APIs of the backend chart repositories - apiClient *ChartClient - - // Point to the url of the backend server - backendServerAddress *url.URL -} - -// GetChartsByNs gets the chart list under the namespace -func (uh *UtilityHandler) GetChartsByNs(namespace string) ([]*ChartInfo, error) { - if len(strings.TrimSpace(namespace)) == 0 { - return nil, errors.New("empty namespace when getting chart list") - } - - path := fmt.Sprintf("/api/%s/charts", namespace) - url := fmt.Sprintf("%s%s", uh.backendServerAddress.String(), path) - - content, err := uh.apiClient.GetContent(url) - if err != nil { - return nil, err - } - - return uh.chartOperator.GetChartList(content) -} - -// DeleteChart deletes all the chart versions of the specified chart under the namespace. -func (uh *UtilityHandler) DeleteChart(namespace, chartName string) error { - if len(strings.TrimSpace(namespace)) == 0 { - return errors.New("empty namespace when deleting chart") - } - - if len(strings.TrimSpace(chartName)) == 0 { - return errors.New("empty chart name when deleting chart") - } - - path := fmt.Sprintf("/api/%s/charts/%s", namespace, chartName) - url := fmt.Sprintf("%s%s", uh.backendServerAddress.String(), path) - - content, err := uh.apiClient.GetContent(url) - if err != nil { - return err - } - - allVersions, err := uh.chartOperator.GetChartVersions(content) - if err != nil { - return err - } - - // Let's delete the versions in parallel - // The number of goroutine is controlled by the const maxDeletionThreads - qSize := len(allVersions) - if qSize > maxDeletionThreads { - qSize = maxDeletionThreads - } - tokenQueue := make(chan struct{}, qSize) - errChan := make(chan error, 1) - waitGroup := new(sync.WaitGroup) - waitGroup.Add(len(allVersions)) - - // Append initial tokens - for i := 0; i < qSize; i++ { - tokenQueue <- struct{}{} - } - - // Collect errors - errs := make([]error, 0) - errWrapper := make(chan error, 1) - go func() { - defer func() { - // pass to the out func - if len(errs) > 0 { - errWrapper <- fmt.Errorf("%v", errs) - } - close(errWrapper) - }() - - for deletionErr := range errChan { - errs = append(errs, deletionErr) - } - }() - - // Schedule deletion tasks - for _, deletingVersion := range allVersions { - // Apply for token first - // If no available token, pending here - <-tokenQueue - - // Got one token - go func(deletingVersion *helm_repo.ChartVersion) { - defer func() { - // return the token back - tokenQueue <- struct{}{} - - // done - waitGroup.Done() - }() - - if err := uh.deleteChartVersion(namespace, chartName, deletingVersion.GetVersion()); err != nil { - errChan <- err - } - }(deletingVersion) - } - - // Wait all goroutines are done - waitGroup.Wait() - // Safe to quit error collection goroutine - close(errChan) - - err = <-errWrapper - - return err -} - -// GetChartVersion returns the summary of the specified chart version. -func (uh *UtilityHandler) GetChartVersion(namespace, name, version string) (*helm_repo.ChartVersion, error) { - if len(namespace) == 0 || len(name) == 0 || len(version) == 0 { - return nil, errors.New("bad arguments to get chart version summary") - } - - path := fmt.Sprintf("/api/%s/charts/%s/%s", namespace, name, version) - url := fmt.Sprintf("%s%s", uh.backendServerAddress.String(), path) - - content, err := uh.apiClient.GetContent(url) - if err != nil { - return nil, err - } - - chartVersion := &helm_repo.ChartVersion{} - if err := yaml.Unmarshal(content, chartVersion); err != nil { - return nil, err - } - - return chartVersion, nil -} - -// deleteChartVersion deletes the specified chart version -func (uh *UtilityHandler) deleteChartVersion(namespace, chartName, version string) error { - path := fmt.Sprintf("/api/%s/charts/%s/%s", namespace, chartName, version) - url := fmt.Sprintf("%s%s", uh.backendServerAddress.String(), path) - - return uh.apiClient.DeleteContent(url) -} diff --git a/src/chartserver/utility_handler_test.go b/src/chartserver/utility_handler_test.go deleted file mode 100644 index 4562800ac..000000000 --- a/src/chartserver/utility_handler_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package chartserver - -import ( - "net/http" - "net/http/httptest" - "net/url" - "testing" -) - -// TestGetChartsByNs tests GetChartsByNs method in UtilityHandler -func TestGetChartsByNs(t *testing.T) { - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.RequestURI { - case "/api/repo1/charts": - if r.Method == http.MethodGet { - w.Write(chartListContent) - return - } - } - - w.WriteHeader(http.StatusNotImplemented) - w.Write([]byte("not supported")) - })) - defer mockServer.Close() - - serverURL, err := url.Parse(mockServer.URL) - if err != nil { - t.Fatal(err) - } - - theController, err := NewController(serverURL) - if err != nil { - t.Fatal(err) - } - - charts, err := theController.GetUtilityHandler().GetChartsByNs("repo1") - if err != nil { - t.Fatal(err) - } - - if len(charts) != 2 { - t.Fatalf("expect 2 items but got %d", len(charts)) - } -} - -// Test the function DeleteChart -func TestDeleteChart(t *testing.T) { - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.RequestURI { - case "/api/repo1/charts/harbor": - if r.Method == http.MethodGet { - w.Write([]byte(chartVersionsOfHarbor)) - return - } - case "/api/repo1/charts/harbor/0.2.0", - "/api/repo1/charts/harbor/0.2.1": - if r.Method == http.MethodDelete { - w.WriteHeader(http.StatusOK) - return - } - } - - w.WriteHeader(http.StatusNotImplemented) - w.Write([]byte("not supported")) - })) - defer mockServer.Close() - - serverURL, err := url.Parse(mockServer.URL) - if err != nil { - t.Fatal(err) - } - - theController, err := NewController(serverURL) - if err != nil { - t.Fatal(err) - } - - if err := theController.GetUtilityHandler().DeleteChart("repo1", "harbor"); err != nil { - t.Fatal(err) - } -} - -// Test the GetChartVersion in utility handler -func TestGetChartVersionSummary(t *testing.T) { - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.RequestURI { - case "/api/repo1/charts/harbor/0.2.0": - if r.Method == http.MethodGet { - w.Write([]byte(chartVersionOfHarbor020)) - return - } - } - - w.WriteHeader(http.StatusNotImplemented) - w.Write([]byte("not supported")) - })) - defer mockServer.Close() - - serverURL, err := url.Parse(mockServer.URL) - if err != nil { - t.Fatal(err) - } - - theController, err := NewController(serverURL) - if err != nil { - t.Fatal(err) - } - - chartV, err := theController.GetUtilityHandler().GetChartVersion("repo1", "harbor", "0.2.0") - if err != nil { - t.Fatal(err) - } - - if chartV.GetName() != "harbor" { - t.Fatalf("expect chart name 'harbor' but got '%s'", chartV.GetName()) - } - - if chartV.GetVersion() != "0.2.0" { - t.Fatalf("expect chart version '0.2.0' but got '%s'", chartV.GetVersion()) - } -} - -var chartVersionOfHarbor020 = ` -{ - "name": "harbor", - "home": "https://github.com/vmware/harbor", - "sources": [ - "https://github.com/vmware/harbor/tree/master/contrib/helm/harbor" - ], - "version": "0.2.0", - "description": "An Enterprise-class Docker Registry by VMware", - "keywords": [ - "vmware", - "docker", - "registry", - "harbor" - ], - "maintainers": [ - { - "name": "Jesse Hu", - "email": "huh@vmware.com" - }, - { - "name": "paulczar", - "email": "username.taken@gmail.com" - } - ], - "engine": "gotpl", - "icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png", - "appVersion": "1.5.0", - "urls": [ - "charts/harbor-0.2.0.tgz" - ], - "created": "2018-08-29T10:26:21.141611102Z", - "digest": "fc8aae8dade9f0dfca12e9f1085081c49843d30a063a3fa7eb42497e3ceb277c" -} -` - -var chartVersionsOfHarbor = ` -[ - { - "name": "harbor", - "home": "https://github.com/vmware/harbor", - "sources": [ - "https://github.com/vmware/harbor/tree/master/contrib/helm/harbor" - ], - "version": "0.2.1", - "description": "An Enterprise-class Docker Registry by VMware", - "keywords": [ - "vmware", - "docker", - "registry", - "harbor" - ], - "maintainers": [ - { - "name": "Jesse Hu", - "email": "huh@vmware.com" - }, - { - "name": "paulczar", - "email": "username.taken@gmail.com" - } - ], - "engine": "gotpl", - "icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png", - "appVersion": "1.5.0", - "urls": [ - "charts/harbor-0.2.1.tgz" - ], - "created": "2018-08-29T10:26:29.625749155Z", - "digest": "2538edf4ddb797af8e025f3bd6226270440110bbdb689bad48656a519a154236" - }, - { - "name": "harbor", - "home": "https://github.com/vmware/harbor", - "sources": [ - "https://github.com/vmware/harbor/tree/master/contrib/helm/harbor" - ], - "version": "0.2.0", - "description": "An Enterprise-class Docker Registry by VMware", - "keywords": [ - "vmware", - "docker", - "registry", - "harbor" - ], - "maintainers": [ - { - "name": "Jesse Hu", - "email": "huh@vmware.com" - }, - { - "name": "paulczar", - "email": "username.taken@gmail.com" - } - ], - "engine": "gotpl", - "icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png", - "appVersion": "1.5.0", - "urls": [ - "charts/harbor-0.2.0.tgz" - ], - "created": "2018-08-29T10:26:21.141611102Z", - "digest": "fc8aae8dade9f0dfca12e9f1085081c49843d30a063a3fa7eb42497e3ceb277c" - } -] -` diff --git a/src/chartserver/utils.go b/src/chartserver/utils.go index 6162d6610..cb9fa6750 100644 --- a/src/chartserver/utils.go +++ b/src/chartserver/utils.go @@ -4,8 +4,8 @@ import ( "encoding/json" "errors" "fmt" - "net/http" "net/url" + "os" "strings" ) @@ -14,30 +14,6 @@ const ( contentTypeJSON = "application/json" ) -// WriteError writes error to http client -func WriteError(w http.ResponseWriter, code int, err error) { - errorObj := make(map[string]string) - errorObj["error"] = err.Error() - errorContent, errorMarshal := json.Marshal(errorObj) - if errorMarshal != nil { - errorContent = []byte(err.Error()) - } - w.WriteHeader(code) - w.Write(errorContent) -} - -// WriteInternalError writes error with statusCode == 500 -func WriteInternalError(w http.ResponseWriter, err error) { - WriteError(w, http.StatusInternalServerError, err) -} - -// Write JSON data to http client -func writeJSONData(w http.ResponseWriter, data []byte) { - w.Header().Set(contentTypeHeader, contentTypeJSON) - w.WriteHeader(http.StatusOK) - w.Write(data) -} - // Extract error object '{"error": "****---***"}' from the content if existing // nil error will be returned if it does exist func extractError(content []byte) error { @@ -116,3 +92,38 @@ func parseRedisConfig(redisConfigV string) (string, error) { return string(cfgData), nil } + +// What's the cache driver if it is set +func parseCacheDriver() (string, bool) { + driver, ok := os.LookupEnv(cacheDriverENVKey) + return strings.ToLower(driver), ok +} + +// Get and parse the configuration for the chart cache +func getCacheConfig() (*ChartCacheConfig, error) { + driver, isSet := parseCacheDriver() + if !isSet { + return nil, nil + } + + if driver != cacheDriverMem && driver != cacheDriverRedis { + return nil, fmt.Errorf("cache driver '%s' is not supported, only support 'memory' and 'redis'", driver) + } + + if driver == cacheDriverMem { + return &ChartCacheConfig{ + DriverType: driver, + }, nil + } + + redisConfigV := os.Getenv(redisENVKey) + redisCfg, err := parseRedisConfig(redisConfigV) + if err != nil { + return nil, fmt.Errorf("failed to parse redis configurations from '%s' with error: %s", redisCfg, err) + } + + return &ChartCacheConfig{ + DriverType: driver, + Config: redisCfg, + }, nil +} diff --git a/src/ui/api/base.go b/src/ui/api/base.go index 2abe521a9..7635dfde4 100644 --- a/src/ui/api/base.go +++ b/src/ui/api/base.go @@ -17,6 +17,7 @@ package api import ( "net/http" + yaml "github.com/ghodss/yaml" "github.com/goharbor/harbor/src/common/api" "github.com/goharbor/harbor/src/common/security" "github.com/goharbor/harbor/src/common/utils/log" @@ -26,6 +27,10 @@ import ( "github.com/goharbor/harbor/src/ui/utils" ) +const ( + yamlFileContentType = "application/x-yaml" +) + // BaseController ... type BaseController struct { api.BaseAPI @@ -98,6 +103,26 @@ func (b *BaseController) SendForbiddenError(err error) { b.RenderFormatedError(http.StatusForbidden, err) } +// WriteJSONData writes the JSON data to the client. +func (b *BaseController) WriteJSONData(object interface{}) { + b.Data["json"] = object + b.ServeJSON() +} + +// WriteYamlData writes the yaml data to the client. +func (b *BaseController) WriteYamlData(object interface{}) { + yData, err := yaml.Marshal(object) + if err != nil { + b.SendInternalServerError(err) + return + } + + w := b.Ctx.ResponseWriter + w.Header().Set("Content-Type", yamlFileContentType) + w.WriteHeader(http.StatusOK) + w.Write(yData) +} + // Init related objects/configurations for the API controllers func Init() error { // If chart repository is not enabled then directly return diff --git a/src/ui/api/chart_label.go b/src/ui/api/chart_label.go index a08bbd567..2b6f697e0 100644 --- a/src/ui/api/chart_label.go +++ b/src/ui/api/chart_label.go @@ -55,7 +55,7 @@ func (cla *ChartLabelAPI) Prepare() { chartName := cla.GetStringFromPath(nameParam) version := cla.GetStringFromPath(versionParam) - if _, err = chartController.GetUtilityHandler().GetChartVersion(project, chartName, version); err != nil { + if _, err = chartController.GetChartVersion(project, chartName, version); err != nil { cla.SendNotFoundError(err) return } diff --git a/src/ui/api/chart_repository.go b/src/ui/api/chart_repository.go index b3652293c..c17e0240f 100644 --- a/src/ui/api/chart_repository.go +++ b/src/ui/api/chart_repository.go @@ -2,7 +2,6 @@ package api import ( "bytes" - "context" "errors" "fmt" "io" @@ -12,6 +11,9 @@ import ( "net/url" "strings" + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/ui/label" + "github.com/goharbor/harbor/src/chartserver" hlog "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/ui/config" @@ -45,6 +47,9 @@ type ChartRepositoryAPI struct { // The base controller to provide common utilities BaseController + // For label management + labelManager *label.BaseManager + // Keep the namespace if existing namespace string } @@ -75,18 +80,19 @@ func (cra *ChartRepositoryAPI) Prepare() { } } - // Rewrite URL path - cra.rewriteURLPath(cra.Ctx.Request) + // Init label manager + cra.labelManager = &label.BaseManager{} } -// GetHealthStatus handles GET /api/chartserver/health +// GetHealthStatus handles GET /api/chartrepo/health func (cra *ChartRepositoryAPI) GetHealthStatus() { // Check access if !cra.requireAccess(cra.namespace, accessLevelSystem) { return } - chartController.GetBaseHandler().GetHealthStatus(cra.Ctx.ResponseWriter, cra.Ctx.Request) + // Directly proxy to the backend + chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request) } // GetIndexByRepo handles GET /:repo/index.yaml @@ -96,7 +102,8 @@ func (cra *ChartRepositoryAPI) GetIndexByRepo() { return } - chartController.GetRepositoryHandler().GetIndexFileWithNS(cra.Ctx.ResponseWriter, cra.Ctx.Request) + // Directly proxy to the backend + chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request) } // GetIndex handles GET /index.yaml @@ -106,7 +113,24 @@ func (cra *ChartRepositoryAPI) GetIndex() { return } - chartController.GetRepositoryHandler().GetIndexFile(cra.Ctx.ResponseWriter, cra.Ctx.Request) + results, err := cra.ProjectMgr.List(nil) + if err != nil { + cra.SendInternalServerError(err) + return + } + + namespaces := []string{} + for _, r := range results.Projects { + namespaces = append(namespaces, r.Name) + } + + indexFile, err := chartController.GetIndexFile(namespaces) + if err != nil { + cra.SendInternalServerError(err) + return + } + + cra.WriteYamlData(indexFile) } // DownloadChart handles GET /:repo/charts/:filename @@ -116,7 +140,8 @@ func (cra *ChartRepositoryAPI) DownloadChart() { return } - chartController.GetRepositoryHandler().DownloadChartObject(cra.Ctx.ResponseWriter, cra.Ctx.Request) + // Directly proxy to the backend + chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request) } // ListCharts handles GET /api/:repo/charts @@ -126,7 +151,13 @@ func (cra *ChartRepositoryAPI) ListCharts() { return } - chartController.GetManipulationHandler().ListCharts(cra.Ctx.ResponseWriter, cra.Ctx.Request) + charts, err := chartController.ListCharts(cra.namespace) + if err != nil { + cra.SendInternalServerError(err) + return + } + + cra.WriteJSONData(charts) } // ListChartVersions GET /api/:repo/charts/:name @@ -136,7 +167,25 @@ func (cra *ChartRepositoryAPI) ListChartVersions() { return } - chartController.GetManipulationHandler().GetChart(cra.Ctx.ResponseWriter, cra.Ctx.Request) + chartName := cra.GetStringFromPath(nameParam) + + versions, err := chartController.GetChart(cra.namespace, chartName) + if err != nil { + cra.SendInternalServerError(err) + return + } + + // Append labels + for _, chartVersion := range versions { + labels, err := cra.labelManager.GetLabelsOfResource(common.ResourceTypeChart, chartFullName(cra.namespace, chartVersion.Name, chartVersion.Version)) + if err != nil { + cra.SendInternalServerError(err) + return + } + chartVersion.Labels = labels + } + + cra.WriteJSONData(versions) } // GetChartVersion handles GET /api/:repo/charts/:name/:version @@ -146,11 +195,25 @@ func (cra *ChartRepositoryAPI) GetChartVersion() { return } - // Let's pass the namespace via the context of request - req := cra.Ctx.Request - *req = *(req.WithContext(context.WithValue(req.Context(), chartserver.NamespaceContextKey, cra.namespace))) + // Get other parameters + chartName := cra.GetStringFromPath(nameParam) + version := cra.GetStringFromPath(versionParam) - chartController.GetManipulationHandler().GetChartVersion(cra.Ctx.ResponseWriter, req) + chartVersion, err := chartController.GetChartVersionDetails(cra.namespace, chartName, version) + if err != nil { + cra.SendInternalServerError(err) + return + } + + // Append labels + labels, err := cra.labelManager.GetLabelsOfResource(common.ResourceTypeChart, chartFullName(cra.namespace, chartName, version)) + if err != nil { + cra.SendInternalServerError(err) + return + } + chartVersion.Labels = labels + + cra.WriteJSONData(chartVersion) } // DeleteChartVersion handles DELETE /api/:repo/charts/:name/:version @@ -160,7 +223,20 @@ func (cra *ChartRepositoryAPI) DeleteChartVersion() { return } - chartController.GetManipulationHandler().DeleteChartVersion(cra.Ctx.ResponseWriter, cra.Ctx.Request) + // Get other parameters + chartName := cra.GetStringFromPath(nameParam) + version := cra.GetStringFromPath(versionParam) + + // Try to remove labels from deleting chart if exitsing + if err := cra.removeLabelsFromChart(chartName, version); err != nil { + cra.SendInternalServerError(err) + return + } + + if err := chartController.DeleteChartVersion(cra.namespace, chartName, version); err != nil { + cra.SendInternalServerError(err) + return + } } // UploadChartVersion handles POST /api/:repo/charts @@ -184,12 +260,13 @@ func (cra *ChartRepositoryAPI) UploadChartVersion() { formField: formFiledNameForProv, }) if err := cra.rewriteFileContent(formFiles, cra.Ctx.Request); err != nil { - chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err) + cra.SendInternalServerError(err) return } } - chartController.GetManipulationHandler().UploadChartVersion(cra.Ctx.ResponseWriter, cra.Ctx.Request) + // Directly proxy to the backend + chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request) } // UploadChartProvFile handles POST /api/:repo/prov @@ -208,12 +285,13 @@ func (cra *ChartRepositoryAPI) UploadChartProvFile() { mustHave: true, }) if err := cra.rewriteFileContent(formFiles, cra.Ctx.Request); err != nil { - chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err) + cra.SendInternalServerError(err) return } } - chartController.GetManipulationHandler().UploadProvenanceFile(cra.Ctx.ResponseWriter, cra.Ctx.Request) + // Directly proxy to the backend + chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request) } // DeleteChart deletes all the chart versions of the specified chart. @@ -226,44 +304,39 @@ func (cra *ChartRepositoryAPI) DeleteChart() { // Get other parameters from the request chartName := cra.GetStringFromPath(nameParam) - if err := chartController.GetUtilityHandler().DeleteChart(cra.namespace, chartName); err != nil { - chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err) + // Remove labels from all the deleting chart versions under the chart + chartVersions, err := chartController.GetChart(cra.namespace, chartName) + if err != nil { + cra.SendInternalServerError(err) + return + } + + for _, chartVersion := range chartVersions { + if err := cra.removeLabelsFromChart(chartName, chartVersion.GetVersion()); err != nil { + cra.SendInternalServerError(err) + return + } + } + + if err := chartController.DeleteChart(cra.namespace, chartName); err != nil { + cra.SendInternalServerError(err) + return } } -// Rewrite the incoming URL with the right backend URL pattern -// Remove 'chartrepo' from the endpoints of manipulation API -// Remove 'chartrepo' from the endpoints of repository services -func (cra *ChartRepositoryAPI) rewriteURLPath(req *http.Request) { - incomingURLPath := req.RequestURI - - defer func() { - hlog.Debugf("Incoming URL '%s' is rewritten to '%s'", incomingURLPath, req.URL.String()) - }() - - // Health check endpoint - if incomingURLPath == chartRepoHealthEndpoint { - req.URL.Path = "/health" - return +func (cra *ChartRepositoryAPI) removeLabelsFromChart(chartName, version string) error { + // Try to remove labels from deleting chart if exitsing + resourceID := chartFullName(cra.namespace, chartName, version) + labels, err := cra.labelManager.GetLabelsOfResource(common.ResourceTypeChart, resourceID) + if err == nil && len(labels) > 0 { + for _, l := range labels { + if err := cra.labelManager.RemoveLabelFromResource(common.ResourceTypeChart, resourceID, l.ID); err != nil { + return err + } + } } - // Root uploading endpoint - if incomingURLPath == rootUploadingEndpoint { - req.URL.Path = strings.Replace(incomingURLPath, "chartrepo", defaultRepo, 1) - return - } - - // Repository endpoints - if strings.HasPrefix(incomingURLPath, "/chartrepo") { - req.URL.Path = strings.TrimPrefix(incomingURLPath, "/chartrepo") - return - } - - // API endpoints - if strings.HasPrefix(incomingURLPath, "/api/chartrepo") { - req.URL.Path = strings.Replace(incomingURLPath, "/chartrepo", "", 1) - return - } + return nil } // Check if there exists a valid namespace @@ -272,20 +345,20 @@ func (cra *ChartRepositoryAPI) rewriteURLPath(req *http.Request) { func (cra *ChartRepositoryAPI) requireNamespace(namespace string) bool { // Actually, never should be like this if len(namespace) == 0 { - cra.HandleBadRequest(":repo should be in the request URL") + cra.SendBadRequestError(errors.New(":repo should be in the request URL")) return false } existsing, err := cra.ProjectMgr.Exists(namespace) if err != nil { // Check failed with error - cra.renderError(http.StatusInternalServerError, fmt.Sprintf("failed to check existence of namespace %s with error: %s", namespace, err.Error())) + cra.SendInternalServerError(fmt.Errorf("failed to check existence of namespace %s with error: %s", namespace, err.Error())) return false } // Not existing if !existsing { - cra.renderError(http.StatusBadRequest, fmt.Sprintf("namespace %s is not existing", namespace)) + cra.SendBadRequestError(fmt.Errorf("namespace %s is not existing", namespace)) return false } @@ -328,7 +401,7 @@ func (cra *ChartRepositoryAPI) requireAccess(namespace string, accessLevel uint) } default: // access rejected for invalid scope - cra.renderError(http.StatusForbidden, "unrecognized access scope") + cra.SendForbiddenError(errors.New("unrecognized access scope")) return false } @@ -336,23 +409,18 @@ func (cra *ChartRepositoryAPI) requireAccess(namespace string, accessLevel uint) if err != nil { // Unauthenticated, return 401 if !cra.SecurityCtx.IsAuthenticated() { - cra.renderError(http.StatusUnauthorized, "Unauthorized") + cra.SendUnAuthorizedError(errors.New("Unauthorized")) return false } // Authenticated, return 403 - cra.renderError(http.StatusForbidden, err.Error()) + cra.SendForbiddenError(err) return false } return true } -// write error message with unified format -func (cra *ChartRepositoryAPI) renderError(code int, text string) { - chartserver.WriteError(cra.Ctx.ResponseWriter, code, errors.New(text)) -} - // formFile is used to represent the uploaded files in the form type formFile struct { // form field key contains the form file @@ -442,3 +510,8 @@ func initializeChartController() (*chartserver.Controller, error) { func isMultipartFormData(req *http.Request) bool { return strings.Contains(req.Header.Get(headerContentType), contentTypeMultipart) } + +// Return the chart full name +func chartFullName(namespace, chartName, version string) string { + return fmt.Sprintf("%s/%s:%s", namespace, chartName, version) +} diff --git a/src/ui/api/chart_repository_test.go b/src/ui/api/chart_repository_test.go index 820c24161..7002a7a6c 100644 --- a/src/ui/api/chart_repository_test.go +++ b/src/ui/api/chart_repository_test.go @@ -9,46 +9,6 @@ import ( "github.com/goharbor/harbor/src/ui/promgr/metamgr" ) -// Test the URL rewrite function -func TestURLRewrite(t *testing.T) { - chartAPI := &ChartRepositoryAPI{} - req, err := createRequest(http.MethodGet, "/api/chartrepo/health") - if err != nil { - t.Fatal(err) - } - chartAPI.rewriteURLPath(req) - if req.URL.Path != "/health" { - t.Fatalf("Expect url format %s but got %s", "/health", req.URL.Path) - } - - req, err = createRequest(http.MethodGet, "/api/chartrepo/library/charts") - if err != nil { - t.Fatal(err) - } - chartAPI.rewriteURLPath(req) - if req.URL.Path != "/api/library/charts" { - t.Fatalf("Expect url format %s but got %s", "/api/library/charts", req.URL.Path) - } - - req, err = createRequest(http.MethodPost, "/api/chartrepo/charts") - if err != nil { - t.Fatal(err) - } - chartAPI.rewriteURLPath(req) - if req.URL.Path != "/api/library/charts" { - t.Fatalf("Expect url format %s but got %s", "/api/library/charts", req.URL.Path) - } - - req, err = createRequest(http.MethodGet, "/chartrepo/library/index.yaml") - if err != nil { - t.Fatal(err) - } - chartAPI.rewriteURLPath(req) - if req.URL.Path != "/library/index.yaml" { - t.Fatalf("Expect url format %s but got %s", "/library/index.yaml", req.URL.Path) - } -} - // Test access checking func TestRequireAccess(t *testing.T) { chartAPI := &ChartRepositoryAPI{} diff --git a/src/ui/api/project.go b/src/ui/api/project.go index 646b46d43..ca141e002 100644 --- a/src/ui/api/project.go +++ b/src/ui/api/project.go @@ -298,7 +298,7 @@ func (p *ProjectAPI) deletable(projectID int64) (*deletableResp, error) { // Check helm charts number if config.WithChartMuseum() { - charts, err := chartController.GetUtilityHandler().GetChartsByNs(p.project.Name) + charts, err := chartController.ListCharts(p.project.Name) if err != nil { return nil, err } diff --git a/src/ui/api/search.go b/src/ui/api/search.go index 957165be3..dbb6085e2 100644 --- a/src/ui/api/search.go +++ b/src/ui/api/search.go @@ -125,7 +125,7 @@ func (s *SearchAPI) Get() { } if searchHandler == nil { - searchHandler = chartController.GetRepositoryHandler().SearchChart + searchHandler = chartController.SearchChart } chartResults, err := searchHandler(keyword, proNames)