From 34f19f437d46e691165da8ee2a689bc6a9b59e54 Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Thu, 30 Aug 2018 11:26:46 +0800 Subject: [PATCH] Delete all the versions of the specified chart - add API route - add DELETE chart method in utility handler of chart controller - add UT case for the newly added methods - update swagger.yaml to refelct the new change Signed-off-by: Steven Zou --- docs/swagger.yaml | 26 ++++++ src/chartserver/chart_operator.go | 14 +++ src/chartserver/client.go | 47 +++++++++-- src/chartserver/utility_handler.go | 103 ++++++++++++++++++++++ src/chartserver/utility_handler_test.go | 108 ++++++++++++++++++++++++ src/ui/api/chart_repository.go | 16 ++++ src/ui/router.go | 1 + 7 files changed, 307 insertions(+), 8 deletions(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 241c773fe..e9b69222f 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2903,6 +2903,32 @@ paths: $ref: '#/definitions/InternalChartAPIError' '200': $ref: '#/definitions/ChartVersions' + delete: + summary: Delete all the versions of the specified chart + description: Delete all the versions of the specified chart + tags: + - Products + - Chart Repository + parameters: + - name: repo + in: path + type: string + required: true + description: The project name + - name: name + in: path + type: string + required: true + description: The chart name + responses: + '401': + $ref: '#/definitions/UnauthorizedChartAPIError' + '403': + $ref: '#/definitions/ForbiddenChartAPIError' + '500': + $ref: '#/definitions/InternalChartAPIError' + '200': + description: The specified chart entry is successfully deleted. /chartrepo/{repo}/charts/{name}/{version}: get: summary: Get the specified chart version diff --git a/src/chartserver/chart_operator.go b/src/chartserver/chart_operator.go index 4908536c8..ac1512eb9 100644 --- a/src/chartserver/chart_operator.go +++ b/src/chartserver/chart_operator.go @@ -138,6 +138,20 @@ func (cho *ChartOperator) GetChartList(content []byte) ([]*ChartInfo, error) { return chartList, nil } +//GetChartVersions returns the chart versions +func (cho *ChartOperator) GetChartVersions(content []byte) (helm_repo.ChartVersions, error) { + if content == nil || len(content) == 0 { + return nil, errors.New("zero content") + } + + chartVersions := make(helm_repo.ChartVersions, 0) + if err := json.Unmarshal(content, &chartVersions); err != nil { + return nil, err + } + + return chartVersions, nil +} + //Get the latest and oldest chart versions func getTheTwoCharts(chartVersions helm_repo.ChartVersions) (latestChart *helm_repo.ChartVersion, oldestChart *helm_repo.ChartVersion) { if len(chartVersions) == 1 { diff --git a/src/chartserver/client.go b/src/chartserver/client.go index 6b96fb18b..ac09c7e1d 100644 --- a/src/chartserver/client.go +++ b/src/chartserver/client.go @@ -3,6 +3,7 @@ package chartserver import ( "errors" "fmt" + "io" "io/ioutil" "net/http" "net/url" @@ -44,6 +45,28 @@ func NewChartClient(credentail *Credential) *ChartClient { //Create http client //GetContent get the bytes from the specified url func (cc *ChartClient) GetContent(addr string) ([]byte, error) { + response, err := cc.sendRequest(addr, http.MethodGet, nil, []int{http.StatusOK}) + if err != nil { + return nil, err + } + + content, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, err + } + defer response.Body.Close() + + return content, nil +} + +//DeleteContent sends deleting request to the addr to delete content +func (cc *ChartClient) DeleteContent(addr string) error { + _, err := cc.sendRequest(addr, http.MethodDelete, nil, []int{http.StatusOK}) + return err +} + +//sendRequest sends requests to the addr with the specified spec +func (cc *ChartClient) sendRequest(addr string, method string, body io.Reader, expectedCodes []int) (*http.Response, error) { if len(strings.TrimSpace(addr)) == 0 { return nil, errors.New("empty url is not allowed") } @@ -53,7 +76,7 @@ func (cc *ChartClient) GetContent(addr string) ([]byte, error) { return nil, fmt.Errorf("invalid url: %s", err.Error()) } - request, err := http.NewRequest(http.MethodGet, addr, nil) + request, err := http.NewRequest(method, addr, body) if err != nil { return nil, err } @@ -68,19 +91,27 @@ func (cc *ChartClient) GetContent(addr string) ([]byte, error) { return nil, err } - content, err := ioutil.ReadAll(response.Body) - if err != nil { - return nil, err + isExpectedStatusCode := false + for _, eCode := range expectedCodes { + if eCode == response.StatusCode { + isExpectedStatusCode = true + break + } } - defer response.Body.Close() - if response.StatusCode != http.StatusOK { + if !isExpectedStatusCode { + content, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, err + } + defer response.Body.Close() + if err := extractError(content); err != nil { return nil, err } - return nil, fmt.Errorf("failed to retrieve content from '%s' with error: %s", fullURI.Path, content) + return nil, fmt.Errorf("%s '%s' failed with error: %s", method, fullURI.Path, content) } - return content, nil + return response, nil } diff --git a/src/chartserver/utility_handler.go b/src/chartserver/utility_handler.go index dc09f83fb..b20f8562f 100644 --- a/src/chartserver/utility_handler.go +++ b/src/chartserver/utility_handler.go @@ -5,6 +5,13 @@ import ( "fmt" "net/url" "strings" + "sync" + + helm_repo "k8s.io/helm/pkg/repo" +) + +const ( + maxDeletionThreads = 10 ) //UtilityHandler provides utility methods @@ -35,3 +42,99 @@ func (uh *UtilityHandler) GetChartsByNs(namespace string) ([]*ChartInfo, error) return uh.chartOperator.GetChartList(content) } + +//DeleteChart deletes all the chart versions of the specified chart under the namespace. +func (uh *UtilityHandler) DeleteChart(namespace, chartName string) error { + if len(strings.TrimSpace(namespace)) == 0 { + return errors.New("empty namespace when deleting chart") + } + + if len(strings.TrimSpace(chartName)) == 0 { + return errors.New("empty chart name when deleting chart") + } + + path := fmt.Sprintf("/api/%s/charts/%s", namespace, chartName) + url := fmt.Sprintf("%s%s", uh.backendServerAddress.String(), path) + + content, err := uh.apiClient.GetContent(url) + if err != nil { + return err + } + + allVersions, err := uh.chartOperator.GetChartVersions(content) + if err != nil { + return err + } + + //Let's delete the versions in parallel + //The number of goroutine is controlled by the const maxDeletionThreads + qSize := len(allVersions) + if qSize > maxDeletionThreads { + qSize = maxDeletionThreads + } + tokenQueue := make(chan struct{}, qSize) + errChan := make(chan error, 1) + waitGroup := new(sync.WaitGroup) + waitGroup.Add(len(allVersions)) + + //Append initial tokens + for i := 0; i < qSize; i++ { + tokenQueue <- struct{}{} + } + + //Collect errors + errs := make([]error, 0) + errWrapper := make(chan error, 1) + go func() { + defer func() { + //pass to the out func + if len(errs) > 0 { + errWrapper <- fmt.Errorf("%v", errs) + } + close(errWrapper) + }() + + for deletionErr := range errChan { + errs = append(errs, deletionErr) + } + }() + + //Schedule deletion tasks + for _, deletingVersion := range allVersions { + //Apply for token first + //If no available token, pending here + <-tokenQueue + + //Got one token + go func(deletingVersion *helm_repo.ChartVersion) { + defer func() { + //return the token back + tokenQueue <- struct{}{} + + //done + waitGroup.Done() + }() + + if err := uh.deleteChartVersion(namespace, chartName, deletingVersion.GetVersion()); err != nil { + errChan <- err + } + }(deletingVersion) + } + + //Wait all goroutines are done + waitGroup.Wait() + //Safe to quit error collection goroutine + close(errChan) + + err = <-errWrapper + + return err +} + +//deleteChartVersion deletes the specified chart version +func (uh *UtilityHandler) deleteChartVersion(namespace, chartName, version string) error { + path := fmt.Sprintf("/api/%s/charts/%s/%s", namespace, chartName, version) + url := fmt.Sprintf("%s%s", uh.backendServerAddress.String(), path) + + return uh.apiClient.DeleteContent(url) +} diff --git a/src/chartserver/utility_handler_test.go b/src/chartserver/utility_handler_test.go index a88dae63c..0fdb43497 100644 --- a/src/chartserver/utility_handler_test.go +++ b/src/chartserver/utility_handler_test.go @@ -42,3 +42,111 @@ func TestGetChartsByNs(t *testing.T) { t.Fatalf("expect 2 items but got %d", len(charts)) } } + +//Test the function DeleteChart +func TestDeleteChart(t *testing.T) { + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.RequestURI { + case "/api/repo1/charts/harbor": + if r.Method == http.MethodGet { + w.Write([]byte(chartVersionsOfHarbor)) + return + } + case "/api/repo1/charts/harbor/0.2.0", + "/api/repo1/charts/harbor/0.2.1": + if r.Method == http.MethodDelete { + w.WriteHeader(http.StatusOK) + return + } + } + + w.WriteHeader(http.StatusNotImplemented) + w.Write([]byte("not supported")) + })) + defer mockServer.Close() + + serverURL, err := url.Parse(mockServer.URL) + if err != nil { + t.Fatal(err) + } + + theController, err := NewController(serverURL) + if err != nil { + t.Fatal(err) + } + + if err := theController.GetUtilityHandler().DeleteChart("repo1", "harbor"); err != nil { + t.Fatal(err) + } +} + +var chartVersionsOfHarbor = ` +[ + { + "name": "harbor", + "home": "https://github.com/vmware/harbor", + "sources": [ + "https://github.com/vmware/harbor/tree/master/contrib/helm/harbor" + ], + "version": "0.2.1", + "description": "An Enterprise-class Docker Registry by VMware", + "keywords": [ + "vmware", + "docker", + "registry", + "harbor" + ], + "maintainers": [ + { + "name": "Jesse Hu", + "email": "huh@vmware.com" + }, + { + "name": "paulczar", + "email": "username.taken@gmail.com" + } + ], + "engine": "gotpl", + "icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png", + "appVersion": "1.5.0", + "urls": [ + "charts/harbor-0.2.1.tgz" + ], + "created": "2018-08-29T10:26:29.625749155Z", + "digest": "2538edf4ddb797af8e025f3bd6226270440110bbdb689bad48656a519a154236" + }, + { + "name": "harbor", + "home": "https://github.com/vmware/harbor", + "sources": [ + "https://github.com/vmware/harbor/tree/master/contrib/helm/harbor" + ], + "version": "0.2.0", + "description": "An Enterprise-class Docker Registry by VMware", + "keywords": [ + "vmware", + "docker", + "registry", + "harbor" + ], + "maintainers": [ + { + "name": "Jesse Hu", + "email": "huh@vmware.com" + }, + { + "name": "paulczar", + "email": "username.taken@gmail.com" + } + ], + "engine": "gotpl", + "icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png", + "appVersion": "1.5.0", + "urls": [ + "charts/harbor-0.2.0.tgz" + ], + "created": "2018-08-29T10:26:21.141611102Z", + "digest": "fc8aae8dade9f0dfca12e9f1085081c49843d30a063a3fa7eb42497e3ceb277c" + } +] +` diff --git a/src/ui/api/chart_repository.go b/src/ui/api/chart_repository.go index b748aaae7..cbbce2605 100644 --- a/src/ui/api/chart_repository.go +++ b/src/ui/api/chart_repository.go @@ -19,6 +19,7 @@ import ( const ( namespaceParam = ":repo" + nameParam = ":name" defaultRepo = "library" rootUploadingEndpoint = "/api/chartrepo/charts" rootIndexEndpoint = "/chartrepo/index.yaml" @@ -215,6 +216,21 @@ func (cra *ChartRepositoryAPI) UploadChartProvFile() { chartController.GetManipulationHandler().UploadProvenanceFile(cra.Ctx.ResponseWriter, cra.Ctx.Request) } +//DeleteChart deletes all the chart versions of the specified chart. +func (cra *ChartRepositoryAPI) DeleteChart() { + //Check access + if !cra.requireAccess(cra.namespace, accessLevelWrite) { + return + } + + //Get other parameters from the request + chartName := cra.GetStringFromPath(nameParam) + + if err := chartController.GetUtilityHandler().DeleteChart(cra.namespace, chartName); err != nil { + chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err) + } +} + //Rewrite the incoming URL with the right backend URL pattern //Remove 'chartrepo' from the endpoints of manipulation API //Remove 'chartrepo' from the endpoints of repository services diff --git a/src/ui/router.go b/src/ui/router.go index f6027f9fb..16ab064ec 100644 --- a/src/ui/router.go +++ b/src/ui/router.go @@ -137,6 +137,7 @@ func initRouters() { beego.Router("/api/chartrepo/health", chartRepositoryAPIType, "get:GetHealthStatus") beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "get:ListCharts") beego.Router("/api/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "get:ListChartVersions") + beego.Router("/api/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "delete:DeleteChart") beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "get:GetChartVersion") beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "delete:DeleteChartVersion") beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "post:UploadChartVersion")