Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-15 23:05:57 +01:00
Code Issues Projects Releases Wiki Activity

Refactor the chart service implementation to provide more extemding flexibilities

Browse Source 
- use unified proxy handler to handle all the proxy traffic
- define interfaces for the handlers
- implement the interface at chart controller level
- refactor UT cases
- update other related reference code

Signed-off-by: Steven Zou <szou@vmware.com>
This commit is contained in:
Steven Zou 2018-09-19 14:45:52 +08:00
parent 8250bce9c5
commit df241c0666
25 changed files with 1141 additions and 1322 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/chartserver/base_handler.go
View File

@ -1,17 +0,0 @@
package chartserver
import (
"net/http"
)
// BaseHandler defines the handlers related with the chart server itself.
type BaseHandler struct {
// Proxy used to to transfer the traffic of requests
// It's mainly used to talk to the backend chart server
trafficProxy *ProxyEngine
}
// GetHealthStatus will return the health status of the backend chart repository server
func (bh *BaseHandler) GetHealthStatus(w http.ResponseWriter, req *http.Request) {
bh.trafficProxy.ServeHTTP(w, req)
}

261
src/chartserver/base_test.go
View File

@ -1,22 +1,14 @@
package chartserver
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"regexp"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/ui/filter"
"github.com/goharbor/harbor/src/ui/promgr/metamgr"
)
// The backend server
var mockServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// The backend server handler
var mockHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.RequestURI {
case "/health":
if r.Method == http.MethodGet {
@ -52,13 +44,27 @@ var mockServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.Respon
w.Write([]byte(harborChartV))
return
}
if r.Method == http.MethodDelete {
w.WriteHeader(http.StatusOK)
return
}
case "/api/repo1/charts/harbor/1.0.0":
if r.Method == http.MethodGet {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Internal error"))
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("Not found"))
return
}
case "/api/repo1/charts/harbor/0.2.1":
if r.Method == http.MethodDelete {
w.WriteHeader(http.StatusOK)
return
}
case "/api/repo1/charts/harbor":
if r.Method == http.MethodGet {
w.Write([]byte(chartVersionsOfHarbor))
return
}
case "/repo3/charts/harbor-0.8.1.tgz":
if r.Method == http.MethodGet {
w.WriteHeader(http.StatusUnauthorized)
w.Write([]byte("Unauthorized"))
@ -68,164 +74,99 @@ var mockServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.Respon
w.WriteHeader(http.StatusNotImplemented)
w.Write([]byte("not supported"))
}))
})
// Chart controller reference
var mockController *Controller
// The frontend server
var frontServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.RequestURI == "/health" && r.Method == http.MethodGet {
mockController.GetBaseHandler().GetHealthStatus(w, r)
return
func createMockObjects() (*httptest.Server, *Controller, error) {
s := httptest.NewServer(mockHandler)
backendURL, err := url.Parse(s.URL)
if err != nil {
s.Close()
return nil, nil, err
}
if r.RequestURI == "/index.yaml" && r.Method == http.MethodGet {
mockProjectMgr := &mockProjectManager{}
*r = *(r.WithContext(context.WithValue(r.Context(), filter.PmKey, mockProjectMgr)))
mockController.GetRepositoryHandler().GetIndexFile(w, r)
return
mockController, err := NewController(backendURL)
if err != nil {
s.Close()
return nil, nil, err
}
var indexYaml = regexp.MustCompile(`^/\w+/index.yaml$`)
if r.Method == http.MethodGet && indexYaml.MatchString(r.RequestURI) {
mockController.GetRepositoryHandler().GetIndexFileWithNS(w, r)
return
}
var downloadFile = regexp.MustCompile(`^/\w+/charts/.+$`)
if r.Method == http.MethodGet && downloadFile.MatchString(r.RequestURI) {
mockController.GetRepositoryHandler().DownloadChartObject(w, r)
return
}
var listCharts = regexp.MustCompile(`^/api/\w+/charts$`)
if r.Method == http.MethodGet && listCharts.MatchString(r.RequestURI) {
mockController.GetManipulationHandler().ListCharts(w, r)
return
}
var getChart = regexp.MustCompile(`^/api/\w+/charts/[\w-]+$`)
if r.Method == http.MethodGet && getChart.MatchString(r.RequestURI) {
mockController.GetManipulationHandler().GetChart(w, r)
return
}
var getChartV = regexp.MustCompile(`^/api/\w+/charts/.+/.+$`)
if r.Method == http.MethodGet && getChartV.MatchString(r.RequestURI) {
*r = *(r.WithContext(context.WithValue(r.Context(), NamespaceContextKey, "repo1")))
mockController.GetManipulationHandler().GetChartVersion(w, r)
return
}
var postCharts = regexp.MustCompile(`^/api/\w+/charts$`)
if r.Method == http.MethodPost && postCharts.MatchString(r.RequestURI) {
mockController.GetManipulationHandler().UploadChartVersion(w, r)
return
}
var postProv = regexp.MustCompile(`^/api/\w+/prov$`)
if r.Method == http.MethodPost && postProv.MatchString(r.RequestURI) {
mockController.GetManipulationHandler().UploadProvenanceFile(w, r)
return
}
var deleteChartV = regexp.MustCompile(`^/api/\w+/charts/.+/.+$`)
if r.Method == http.MethodDelete && deleteChartV.MatchString(r.RequestURI) {
mockController.GetManipulationHandler().DeleteChartVersion(w, r)
return
}
w.WriteHeader(http.StatusNotImplemented)
w.Write([]byte(fmt.Sprintf("no handler to handle %s", r.RequestURI)))
}))
return s, mockController, nil
}
// Http client
var httpClient = NewChartClient(nil)
// Mock project manager
type mockProjectManager struct{}
func (mpm *mockProjectManager) Get(projectIDOrName interface{}) (*models.Project, error) {
return nil, errors.New("Not implemented")
}
func (mpm *mockProjectManager) Create(*models.Project) (int64, error) {
return -1, errors.New("Not implemented")
}
func (mpm *mockProjectManager) Delete(projectIDOrName interface{}) error {
return errors.New("Not implemented")
}
func (mpm *mockProjectManager) Update(projectIDOrName interface{}, project *models.Project) error {
return errors.New("Not implemented")
}
func (mpm *mockProjectManager) List(query *models.ProjectQueryParam) (*models.ProjectQueryResult, error) {
results := &models.ProjectQueryResult{
Total: 2,
Projects: make([]*models.Project, 0),
}
results.Projects = append(results.Projects, &models.Project{ProjectID: 0, Name: "repo1"})
results.Projects = append(results.Projects, &models.Project{ProjectID: 1, Name: "repo2"})
return results, nil
}
func (mpm *mockProjectManager) IsPublic(projectIDOrName interface{}) (bool, error) {
return false, errors.New("Not implemented")
}
func (mpm *mockProjectManager) Exists(projectIDOrName interface{}) (bool, error) {
return false, errors.New("Not implemented")
}
// get all public project
func (mpm *mockProjectManager) GetPublic() ([]*models.Project, error) {
return nil, errors.New("Not implemented")
}
// if the project manager uses a metadata manager, return it, otherwise return nil
func (mpm *mockProjectManager) GetMetadataManager() metamgr.ProjectMetadataManager {
return nil
}
// Utility methods
// Start the mock frontend and backend servers
func startMockServers() error {
mockServer.Start()
backendURL, err := url.Parse(getTheAddrOfMockServer())
if err != nil {
return err
}
mockController, err = NewController(backendURL)
if err != nil {
return err
}
frontServer.Start()
return nil
}
// Stop the mock frontend and backend servers
func stopMockServers() {
frontServer.Close()
mockServer.Close()
}
// Get the address of the backend mock server
func getTheAddrOfMockServer() string {
return mockServer.URL
}
// Get the address of the frontend server
func getTheAddrOfFrontServer() string {
return frontServer.URL
}
// Mock content
var chartVersionsOfHarbor = `
[
{
"name": "harbor",
"home": "https://github.com/vmware/harbor",
"sources": [
"https://github.com/vmware/harbor/tree/master/contrib/helm/harbor"
],
"version": "0.2.1",
"description": "An Enterprise-class Docker Registry by VMware",
"keywords": [
"vmware",
"docker",
"registry",
"harbor"
],
"maintainers": [
{
"name": "Jesse Hu",
"email": "huh@vmware.com"
},
{
"name": "paulczar",
"email": "username.taken@gmail.com"
}
],
"engine": "gotpl",
"icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png",
"appVersion": "1.5.0",
"urls": [
"charts/harbor-0.2.1.tgz"
],
"created": "2018-08-29T10:26:29.625749155Z",
"digest": "2538edf4ddb797af8e025f3bd6226270440110bbdb689bad48656a519a154236"
},
{
"name": "harbor",
"home": "https://github.com/vmware/harbor",
"sources": [
"https://github.com/vmware/harbor/tree/master/contrib/helm/harbor"
],
"version": "0.2.0",
"description": "An Enterprise-class Docker Registry by VMware",
"keywords": [
"vmware",
"docker",
"registry",
"harbor"
],
"maintainers": [
{
"name": "Jesse Hu",
"email": "huh@vmware.com"
},
{
"name": "paulczar",
"email": "username.taken@gmail.com"
}
],
"engine": "gotpl",
"icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png",
"appVersion": "1.5.0",
"urls": [
"charts/harbor-0.2.0.tgz"
],
"created": "2018-08-29T10:26:21.141611102Z",
"digest": "fc8aae8dade9f0dfca12e9f1085081c49843d30a063a3fa7eb42497e3ceb277c"
}
]
`
var harborChartV = `
{
"name": "harbor",

15
src/chartserver/chart_operator.go
View File

@ -11,6 +11,7 @@ import (
"github.com/Masterminds/semver"
"github.com/goharbor/harbor/src/common/models"
hlog "github.com/goharbor/harbor/src/common/utils/log"
"k8s.io/helm/pkg/chartutil"
helm_repo "k8s.io/helm/pkg/repo"
@ -21,6 +22,15 @@ const (
valuesFileName = "values.yaml"
)
// ChartVersion extends the helm ChartVersion with additional labels
type ChartVersion struct {
helm_repo.ChartVersion
Labels []*models.Label `json:"labels"`
}
// ChartVersions is an array of extended ChartVersion
type ChartVersions []*ChartVersion
// ChartVersionDetails keeps the detailed data info of the chart version
type ChartVersionDetails struct {
Metadata *helm_repo.ChartVersion `json:"metadata"`
@ -28,6 +38,7 @@ type ChartVersionDetails struct {
Values map[string]interface{} `json:"values"`
Files map[string]string `json:"files"`
Security *SecurityReport `json:"security"`
Labels []*models.Label `json:"labels"`
}
// SecurityReport keeps the info related with security
@ -154,12 +165,12 @@ func (cho *ChartOperator) GetChartList(content []byte) ([]*ChartInfo, error) {
}
// GetChartVersions returns the chart versions
func (cho *ChartOperator) GetChartVersions(content []byte) (helm_repo.ChartVersions, error) {
func (cho *ChartOperator) GetChartVersions(content []byte) (ChartVersions, error) {
if content == nil || len(content) == 0 {
return nil, errors.New("zero content")
}
chartVersions := make(helm_repo.ChartVersions, 0)
chartVersions := make(ChartVersions, 0)
if err := json.Unmarshal(content, &chartVersions); err != nil {
return nil, err
}

118
src/chartserver/controller.go
View File

@ -5,7 +5,6 @@ import (
"fmt"
"net/url"
"os"
"strings"
hlog "github.com/goharbor/harbor/src/common/utils/log"
)
@ -25,20 +24,21 @@ type Credential struct {
// A reverse proxy will be created and managed to proxy the related traffics between API and
// backend chart server
type Controller struct {
// Proxy used to to transfer the traffic of requests
// It's mainly used to talk to the backend chart server
trafficProxy *ProxyEngine
// Parse and process the chart version to provide required info data
chartOperator *ChartOperator
// HTTP client used to call the realted APIs of the backend chart repositories
apiClient *ChartClient
// The access endpoint of the backend chart repository server
backendServerAddr *url.URL
backendServerAddress *url.URL
// To cover the server info and status requests
baseHandler *BaseHandler
// To cover the chart repository requests
repositoryHandler *RepositoryHandler
// To cover all the manipulation requests
manipulationHandler *ManipulationHandler
// To cover the other utility requests
utilityHandler *UtilityHandler
// Cache the chart data
chartCache *ChartCache
}
// NewController is constructor of the chartserver.Controller
@ -53,15 +53,6 @@ func NewController(backendServer *url.URL) (*Controller, error) {
Password: os.Getenv(passwordKey),
}
// Use customized reverse proxy
proxy := NewProxyEngine(backendServer, cred)
// Create http client with customized timeouts
client := NewChartClient(cred)
// Initialize chart operator for use
operator := &ChartOperator{}
// Creat cache
cacheCfg, err := getCacheConfig()
if err != nil {
@ -75,79 +66,18 @@ func NewController(backendServer *url.URL) (*Controller, error) {
}
return &Controller{
backendServerAddr: backendServer,
baseHandler: &BaseHandler{proxy},
repositoryHandler: &RepositoryHandler{
trafficProxy: proxy,
apiClient: client,
backendServerAddress: backendServer,
},
manipulationHandler: &ManipulationHandler{
trafficProxy: proxy,
chartOperator: operator,
apiClient: client,
backendServerAddress: backendServer,
chartCache: cache,
},
utilityHandler: &UtilityHandler{
apiClient: client,
backendServerAddress: backendServer,
chartOperator: operator,
},
backendServerAddress: backendServer,
// Use customized reverse proxy
trafficProxy: NewProxyEngine(backendServer, cred),
// Initialize chart operator for use
chartOperator: &ChartOperator{},
// Create http client with customized timeouts
apiClient: NewChartClient(cred),
chartCache: cache,
}, nil
}
// GetBaseHandler returns the reference of BaseHandler
func (c *Controller) GetBaseHandler() *BaseHandler {
return c.baseHandler
}
// GetRepositoryHandler returns the reference of RepositoryHandler
func (c *Controller) GetRepositoryHandler() *RepositoryHandler {
return c.repositoryHandler
}
// GetManipulationHandler returns the reference of ManipulationHandler
func (c *Controller) GetManipulationHandler() *ManipulationHandler {
return c.manipulationHandler
}
// GetUtilityHandler returns the reference of UtilityHandler
func (c *Controller) GetUtilityHandler() *UtilityHandler {
return c.utilityHandler
}
// What's the cache driver if it is set
func parseCacheDriver() (string, bool) {
driver, ok := os.LookupEnv(cacheDriverENVKey)
return strings.ToLower(driver), ok
}
// Get and parse the configuration for the chart cache
func getCacheConfig() (*ChartCacheConfig, error) {
driver, isSet := parseCacheDriver()
if !isSet {
return nil, nil
}
if driver != cacheDriverMem && driver != cacheDriverRedis {
return nil, fmt.Errorf("cache driver '%s' is not supported, only support 'memory' and 'redis'", driver)
}
if driver == cacheDriverMem {
return &ChartCacheConfig{
DriverType: driver,
}, nil
}
redisConfigV := os.Getenv(redisENVKey)
redisCfg, err := parseRedisConfig(redisConfigV)
if err != nil {
return nil, fmt.Errorf("failed to parse redis configurations from '%s' with error: %s", redisCfg, err)
}
return &ChartCacheConfig{
DriverType: driver,
Config: redisCfg,
}, nil
// APIPrefix returns the API prefix path of calling backend chart service.
func (c *Controller) APIPrefix(namespace string) string {
return fmt.Sprintf("%s/api/%s/charts", c.backendServerAddress.String(), namespace)
}

220
src/chartserver/controller_test.go
View File

@ -1,225 +1,21 @@
package chartserver
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings"
"testing"
"github.com/ghodss/yaml"
helm_repo "k8s.io/helm/pkg/repo"
)
// Prepare, start the mock servers
func TestStartServers(t *testing.T) {
if err := startMockServers(); err != nil {
t.Fatal(err)
}
}
// Test /health
func TestGetHealthOfBaseHandler(t *testing.T) {
content, err := httpClient.GetContent(fmt.Sprintf("%s/health", getTheAddrOfFrontServer()))
// Test controller
func TestController(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
status := make(map[string]interface{})
if err := json.Unmarshal(content, &status); err != nil {
t.Fatalf("Unmarshal error: %s, %s", err, content)
}
healthy, ok := status["health"].(bool)
if !ok || !healthy {
t.Fatalf("Expect healthy of server to be 'true' but got %v", status["health"])
prefix := c.APIPrefix("fake")
expected := fmt.Sprintf("%s/api/%s/charts", s.URL, "fake")
if prefix != expected {
t.Fatalf("expect '%s' but got '%s'", expected, prefix)
}
}
// Get /repo1/index.yaml
func TestGetIndexYamlByRepo(t *testing.T) {
indexFile, err := getIndexYaml("/repo1/index.yaml")
if err != nil {
t.Fatal(err)
}
if len(indexFile.Entries) != 3 {
t.Fatalf("Expect index file with 3 entries, but got %d", len(indexFile.Entries))
}
}
// Test get /index.yaml
func TestGetUnifiedYamlFile(t *testing.T) {
indexFile, err := getIndexYaml("/index.yaml")
if err != nil {
t.Fatal(err)
}
if len(indexFile.Entries) != 5 {
t.Fatalf("Expect index file with 5 entries, but got %d", len(indexFile.Entries))
}
_, ok := indexFile.Entries["repo1/harbor"]
if !ok {
t.Fatal("Expect chart entry 'repo1/harbor' but got nothing")
}
_, ok = indexFile.Entries["repo2/harbor"]
if !ok {
t.Fatal("Expect chart entry 'repo2/harbor' but got nothing")
}
}
// Test download /:repo/charts/chart.tar
// Use this case to test the proxy function
func TestDownloadChart(t *testing.T) {
content, err := httpClient.GetContent(fmt.Sprintf("%s/repo1/charts/harbor-0.2.0.tgz", getTheAddrOfFrontServer()))
if err != nil {
t.Fatal(err)
}
gotSize := len(content)
expectSize := len(helmChartContent)
if gotSize != expectSize {
t.Fatalf("Expect %d bytes data but got %d bytes", expectSize, gotSize)
}
}
// Test get /api/:repo/charts
func TestRetrieveChartList(t *testing.T) {
content, err := httpClient.GetContent(fmt.Sprintf("%s/api/repo1/charts", getTheAddrOfFrontServer()))
if err != nil {
t.Fatal(err)
}
chartList := make([]*ChartInfo, 0)
err = json.Unmarshal(content, &chartList)
if err != nil {
t.Fatalf("Unmarshal error: %s", err)
}
if len(chartList) != 2 {
t.Fatalf("Expect to get 2 charts in the list but got %d", len(chartList))
}
foundItem := false
for _, chartInfo := range chartList {
if chartInfo.Name == "hello-helm" && chartInfo.TotalVersions == 2 {
foundItem = true
break
}
}
if !foundItem {
t.Fatalf("Expect chart 'hello-helm' with 2 versions but got nothing")
}
}
// Test get /api/:repo/charts/:chart_name/:version
func TestGetChartVersion(t *testing.T) {
content, err := httpClient.GetContent(fmt.Sprintf("%s/api/repo1/charts/harbor/0.2.0", getTheAddrOfFrontServer()))
if err != nil {
t.Fatal(err)
}
chartVersion := &ChartVersionDetails{}
if err = json.Unmarshal(content, chartVersion); err != nil {
t.Fatalf("Unmarshal error: %s", err)
}
if chartVersion.Metadata.Name != "harbor" {
t.Fatalf("Expect harbor chart version but got %s", chartVersion.Metadata.Name)
}
if chartVersion.Metadata.Version != "0.2.0" {
t.Fatalf("Expect version '0.2.0' but got version %s", chartVersion.Metadata.Version)
}
if len(chartVersion.Dependencies) != 1 {
t.Fatalf("Expect 1 dependency but got %d ones", len(chartVersion.Dependencies))
}
if len(chartVersion.Values) != 99 {
t.Fatalf("Expect 99 k-v values but got %d", len(chartVersion.Values))
}
}
// Test get /api/:repo/charts/:chart_name/:version with none-existing version
func TestGetChartVersionWithError(t *testing.T) {
_, err := httpClient.GetContent(fmt.Sprintf("%s/api/repo1/charts/harbor/1.0.0", getTheAddrOfFrontServer()))
if err == nil {
t.Fatal("Expect an error but got nil")
}
}
// Get /api/repo1/charts/harbor
// 401 will be rewritten to 500 with specified error
func TestResponseRewrite(t *testing.T) {
response, err := http.Get(fmt.Sprintf("%s/api/repo1/charts/harbor", getTheAddrOfFrontServer()))
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusInternalServerError {
t.Fatalf("Expect status code 500 but got %d", response.StatusCode)
}
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
t.Fatalf("Read bytes from http response failed with error: %s", err)
}
defer response.Body.Close()
errObj := make(map[string]interface{})
if err = json.Unmarshal(bytes, &errObj); err != nil {
t.Fatalf("Unmarshal error: %s", err)
}
if msg, ok := errObj["error"]; !ok {
t.Fatal("Expect an error message from server but got nothing")
} else {
if !strings.Contains(msg.(string), "operation request from unauthorized source is rejected") {
t.Fatal("Missing the required error message")
}
}
}
// Test the chart searching
func TestChartSearching(t *testing.T) {
namespaces := []string{"repo1", "repo2"}
q := "harbor"
results, err := mockController.GetRepositoryHandler().SearchChart(q, namespaces)
if err != nil {
t.Fatalf("expect nil error but got '%s'", err)
}
if len(results) != 2 {
t.Fatalf("expect 2 results but got %d", len(results))
}
}
// Clear environments
func TestStopServers(t *testing.T) {
stopMockServers()
}
// Utility method for getting index yaml file
func getIndexYaml(path string) (*helm_repo.IndexFile, error) {
content, err := httpClient.GetContent(fmt.Sprintf("%s%s", getTheAddrOfFrontServer(), path))
if err != nil {
return nil, err
}
indexFile := &helm_repo.IndexFile{}
if err := yaml.Unmarshal(content, indexFile); err != nil {
return nil, fmt.Errorf("Unmarshal error: %s", err)
}
if indexFile == nil {
return nil, fmt.Errorf("Got nil index yaml file")
}
return indexFile, nil
}

89
src/chartserver/handler_interface.go Normal file
View File

@ -0,0 +1,89 @@
package chartserver
import (
"net/http"
"k8s.io/helm/cmd/helm/search"
helm_repo "k8s.io/helm/pkg/repo"
)
// ServiceHandler defines the related methods to handle kinds of chart service requests.
type ServiceHandler interface {
// ListCharts lists all the charts under the specified namespace.
//
// namespace string: the chart namespace.
//
// If succeed, a chart info list with nil error will be returned;
// otherwise, a non-nil error will be got.
ListCharts(namespace string) ([]*ChartInfo, error)
// Get all the chart versions of the specified chart under the namespace.
//
// namespace string: the chart namespace.
// chartName string: the name of the chart, e.g: "harbor"
//
// If succeed, a chart version list with nil error will be returned;
// otherwise, a non-nil error will be got.
GetChart(namespace, chartName string) (helm_repo.ChartVersions, error)
// Get the detailed info of the specified chart version under the namespace.
// The detailed info includes chart summary, dependencies, values and signature status etc.
//
// namespace string: the chart namespace.
// chartName string: the name of the chart, e.g: "harbor"
// version string: the SemVer version of the chart, e.g: "0.2.0"
//
// If succeed, chart version details with nil error will be returned;
// otherwise, a non-nil error will be got.
GetChartVersionDetails(namespace, chartName, version string) (*ChartVersionDetails, error)
// SearchChart search charts in the specified namespaces with the keyword q.
// RegExp mode is enabled as default.
// For each chart, only the latest version will shown in the result list if matched to avoid duplicated entries.
// Keep consistent with `helm search` command.
//
// q string : the searching keyword
// namespaces []string : the search namespace scope
//
// If succeed, a search result list with nil error will be returned;
// otherwise, a non-nil error will be got.
SearchChart(q string, namespaces []string) ([]*search.Result, error)
// GetIndexFile will read the index.yaml under all namespaces and merge them as a single one
// Please be aware that, to support this function, the backend chart repository server should
// enable multi-tenancies
//
// namespaces []string : all the namespaces with accessing permissions
//
// If succeed, a unified merged index file with nil error will be returned;
// otherwise, a non-nil error will be got.
GetIndexFile(namespaces []string) (*helm_repo.IndexFile, error)
// Get the chart summary of the specified chart version.
//
// namespace string: the chart namespace.
// chartName string: the name of the chart, e.g: "harbor"
// version string: the SemVer version of the chart, e.g: "0.2.0"
//
// If succeed, chart version summary with nil error will be returned;
// otherwise, a non-nil error will be got.
GetChartVersion(namespace, name, version string) (*helm_repo.ChartVersion, error)
// DeleteChart deletes all the chart versions of the specified chart under the namespace.
//
// namespace string: the chart namespace.
// chartName string: the name of the chart, e.g: "harbor"
//
// If succeed, a nil error will be returned;
// otherwise, a non-nil error will be got.
DeleteChart(namespace, chartName string) error
}
// ProxyTrafficHandler defines the handler methods to handle the proxy traffic.
type ProxyTrafficHandler interface {
// Proxy the traffic to the backended server
//
// Req *http.Request : The incoming http request
// w http.ResponseWriter : The response writer reference
ProxyTraffic(w http.ResponseWriter, req *http.Request)
}

93
src/chartserver/handler_manipulation.go Normal file
View File

@ -0,0 +1,93 @@
package chartserver
import (
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/ghodss/yaml"
helm_repo "k8s.io/helm/pkg/repo"
)
// ListCharts gets the chart list under the namespace
// See @ServiceHandler.ListCharts
func (c *Controller) ListCharts(namespace string) ([]*ChartInfo, error) {
if len(strings.TrimSpace(namespace)) == 0 {
return nil, errors.New("empty namespace when getting chart list")
}
content, err := c.apiClient.GetContent(c.APIPrefix(namespace))
if err != nil {
return nil, err
}
return c.chartOperator.GetChartList(content)
}
// GetChart returns all the chart versions under the specified chart
// See @ServiceHandler.GetChart
func (c *Controller) GetChart(namespace, chartName string) (ChartVersions, error) {
if len(namespace) == 0 {
return nil, errors.New("empty name when getting chart versions")
}
if len(chartName) == 0 {
return nil, errors.New("no chart name specified")
}
url := fmt.Sprintf("%s/%s", c.APIPrefix(namespace), chartName)
data, err := c.apiClient.GetContent(url)
if err != nil {
return nil, err
}
versions := make(ChartVersions, 0)
if err := json.Unmarshal(data, &versions); err != nil {
return nil, err
}
return versions, nil
}
// DeleteChartVersion will delete the specified version of the chart
// See @ServiceHandler.DeleteChartVersion
func (c *Controller) DeleteChartVersion(namespace, chartName, version string) error {
if len(namespace) == 0 {
return errors.New("empty namespace when deleting chart version")
}
if len(chartName) == 0 || len(version) == 0 {
return errors.New("invalid chart for deleting")
}
url := fmt.Sprintf("%s/%s/%s", c.APIPrefix(namespace), chartName, version)
return c.apiClient.DeleteContent(url)
}
// GetChartVersion returns the summary of the specified chart version.
// See @ServiceHandler.GetChartVersion
func (c *Controller) GetChartVersion(namespace, name, version string) (*helm_repo.ChartVersion, error) {
if len(namespace) == 0 {
return nil, errors.New("empty namespace when getting summary of chart version")
}
if len(name) == 0 || len(version) == 0 {
return nil, errors.New("invalid chart when getting summary")
}
url := fmt.Sprintf("%s/%s/%s", c.APIPrefix(namespace), name, version)
content, err := c.apiClient.GetContent(url)
if err != nil {
return nil, err
}
chartVersion := &helm_repo.ChartVersion{}
if err := yaml.Unmarshal(content, chartVersion); err != nil {
return nil, err
}
return chartVersion, nil
}

88
src/chartserver/handler_manipulation_test.go Normal file
View File

@ -0,0 +1,88 @@
package chartserver
import (
"testing"
)
// Test get /api/:repo/charts/harbor
func TestGetChart(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
versions, err := c.GetChart("repo1", "harbor")
if err != nil {
t.Fatal(err)
}
if len(versions) != 2 {
t.Fatalf("expect 2 chart versions of harbor but got %d", len(versions))
}
}
// Test delete /api/:repo/charts/harbor/0.2.0
func TestDeleteChartVersion(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
if err := c.DeleteChartVersion("repo1", "harbor", "0.2.0"); err != nil {
t.Fatal(err)
}
}
// Test get /api/:repo/charts
func TestRetrieveChartList(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
chartList, err := c.ListCharts("repo1")
if err != nil {
t.Fatal(err)
}
if len(chartList) != 2 {
t.Fatalf("Expect to get 2 charts in the list but got %d", len(chartList))
}
foundItem := false
for _, chartInfo := range chartList {
if chartInfo.Name == "hello-helm" && chartInfo.TotalVersions == 2 {
foundItem = true
break
}
}
if !foundItem {
t.Fatalf("Expect chart 'hello-helm' with 2 versions but got nothing")
}
}
// Test the GetChartVersion in utility handler
func TestGetChartVersionSummary(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
chartV, err := c.GetChartVersion("repo1", "harbor", "0.2.0")
if err != nil {
t.Fatal(err)
}
if chartV.GetName() != "harbor" {
t.Fatalf("expect chart name 'harbor' but got '%s'", chartV.GetName())
}
if chartV.GetVersion() != "0.2.0" {
t.Fatalf("expect chart version '0.2.0' but got '%s'", chartV.GetVersion())
}
}

12
src/chartserver/handler_proxy_traffic.go Normal file
View File

@ -0,0 +1,12 @@
package chartserver
import (
"net/http"
)
// ProxyTraffic implements the interface method.
func (c *Controller) ProxyTraffic(w http.ResponseWriter, req *http.Request) {
if c.trafficProxy != nil {
c.trafficProxy.ServeHTTP(w, req)
}
}

136
src/chartserver/handler_proxy_traffic_test.go Normal file
View File

@ -0,0 +1,136 @@
package chartserver
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/ghodss/yaml"
helm_repo "k8s.io/helm/pkg/repo"
)
// The frontend server
var frontServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mockController.ProxyTraffic(w, r)
}))
var mockServer *httptest.Server
var mockController *Controller
// Prepare case
func TestStartMockServers(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
mockController = c
mockServer = s
frontServer.Start()
}
// Test /health
func TestGetHealthOfBaseHandler(t *testing.T) {
content, err := httpClient.GetContent(fmt.Sprintf("%s/api/chartrepo/health", frontServer.URL))
if err != nil {
t.Fatal(err)
}
status := make(map[string]interface{})
if err := json.Unmarshal(content, &status); err != nil {
t.Fatalf("Unmarshal error: %s, %s", err, content)
}
healthy, ok := status["health"].(bool)
if !ok || !healthy {
t.Fatalf("Expect healthy of server to be 'true' but got %v", status["health"])
}
}
// Get /repo1/index.yaml
func TestGetIndexYamlByRepo(t *testing.T) {
indexFile, err := getIndexYaml("/chartrepo/repo1/index.yaml")
if err != nil {
t.Fatal(err)
}
if len(indexFile.Entries) != 3 {
t.Fatalf("Expect index file with 3 entries, but got %d", len(indexFile.Entries))
}
}
// Test download /:repo/charts/chart.tar
// Use this case to test the proxy function
func TestDownloadChart(t *testing.T) {
content, err := httpClient.GetContent(fmt.Sprintf("%s/chartrepo/repo1/charts/harbor-0.2.0.tgz", frontServer.URL))
if err != nil {
t.Fatal(err)
}
gotSize := len(content)
expectSize := len(helmChartContent)
if gotSize != expectSize {
t.Fatalf("Expect %d bytes data but got %d bytes", expectSize, gotSize)
}
}
// Get /api/repo1/charts/harbor
// 401 will be rewritten to 500 with specified error
func TestResponseRewrite(t *testing.T) {
response, err := http.Get(fmt.Sprintf("%s/chartrepo/repo3/charts/harbor-0.8.1.tgz", frontServer.URL))
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusInternalServerError {
t.Fatalf("Expect status code 500 but got %d", response.StatusCode)
}
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
t.Fatalf("Read bytes from http response failed with error: %s", err)
}
defer response.Body.Close()
errObj := make(map[string]interface{})
if err = json.Unmarshal(bytes, &errObj); err != nil {
t.Fatalf("Unmarshal error: %s", err)
}
if msg, ok := errObj["error"]; !ok {
t.Fatal("Expect an error message from server but got nothing")
} else {
if !strings.Contains(msg.(string), "operation request from unauthorized source is rejected") {
t.Fatal("Missing the required error message")
}
}
}
// Clear env
func TestStopMockServers(t *testing.T) {
frontServer.Close()
mockServer.Close()
}
// Utility method for getting index yaml file
func getIndexYaml(path string) (*helm_repo.IndexFile, error) {
content, err := httpClient.GetContent(fmt.Sprintf("%s%s", frontServer.URL, path))
if err != nil {
return nil, err
}
indexFile := &helm_repo.IndexFile{}
if err := yaml.Unmarshal(content, indexFile); err != nil {
return nil, fmt.Errorf("Unmarshal error: %s", err)
}
if indexFile == nil {
return nil, fmt.Errorf("Got nil index yaml file")
}
return indexFile, nil
}

133
src/chartserver/repo_handler.go → src/chartserver/handler_repo.go
View File

@ -1,17 +1,12 @@
package chartserver
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"path"
"sync"
"time"
"github.com/ghodss/yaml"
"github.com/goharbor/harbor/src/ui/filter"
"k8s.io/helm/cmd/helm/search"
helm_repo "k8s.io/helm/pkg/repo"
hlog "github.com/goharbor/harbor/src/common/utils/log"
@ -24,124 +19,33 @@ const (
searchMaxScore = 25
)
// RepositoryHandler defines all the handlers to handle the requests related with chart repository
// e.g: index.yaml and downloading chart objects
type RepositoryHandler struct {
// Proxy used to to transfer the traffic of requests
// It's mainly used to talk to the backend chart server
trafficProxy *ProxyEngine
// HTTP client used to call the realted APIs of the backend chart repositories
apiClient *ChartClient
// Point to the url of the backend server
backendServerAddress *url.URL
}
// Result returned by worker
type processedResult struct {
namespace string
indexFileOfRepo *helm_repo.IndexFile
}
// GetIndexFileWithNS will read the index.yaml data under the specified namespace
func (rh *RepositoryHandler) GetIndexFileWithNS(w http.ResponseWriter, req *http.Request) {
rh.trafficProxy.ServeHTTP(w, req)
}
// GetIndexFile will read the index.yaml under all namespaces and merge them as a single one
// Please be aware that, to support this function, the backend chart repository server should
// enable multi-tenancies
func (rh *RepositoryHandler) GetIndexFile(w http.ResponseWriter, req *http.Request) {
// Get project manager references
projectMgr, err := filter.GetProjectManager(req)
if err != nil {
WriteInternalError(w, err)
return
//
// See @ServiceHandler.GetIndexFile
func (c *Controller) GetIndexFile(namespaces []string) (*helm_repo.IndexFile, error) {
if namespaces == nil || len(namespaces) == 0 {
return emptyIndexFile(), nil
}
// Get all the projects
results, err := projectMgr.List(nil)
if err != nil {
WriteInternalError(w, err)
return
}
// If no projects existing, return empty index.yaml content immediately
if results.Total == 0 {
w.Write(emptyIndexFile())
return
}
namespaces := []string{}
for _, p := range results.Projects {
namespaces = append(namespaces, p.Name)
}
mergedIndexFile, err := rh.getIndexYaml(namespaces)
if err != nil {
WriteInternalError(w, err)
return
}
bytes, err := yaml.Marshal(mergedIndexFile)
if err != nil {
WriteInternalError(w, err)
return
}
w.Write(bytes)
}
// DownloadChartObject will download the stored chart object to the client
// e.g: helm install
func (rh *RepositoryHandler) DownloadChartObject(w http.ResponseWriter, req *http.Request) {
rh.trafficProxy.ServeHTTP(w, req)
}
// SearchChart search charts in the specified namespaces with the keyword q.
// RegExp mode is enabled as default.
// For each chart, only the latest version will shown in the result list if matched to avoid duplicated entries.
// Keep consistent with `helm search` command.
func (rh *RepositoryHandler) SearchChart(q string, namespaces []string) ([]*search.Result, error) {
if len(q) == 0 || len(namespaces) == 0 {
// Return empty list
return []*search.Result{}, nil
}
// Get the merged index yaml file of the namespaces
ind, err := rh.getIndexYaml(namespaces)
if err != nil {
return nil, err
}
// Build the search index
index := search.NewIndex()
// As the repo name is already merged into the index yaml, we use empty repo name.
// Set 'All' to false to return only one version for each chart.
index.AddRepo("", ind, false)
// Search
// RegExp is enabled
results, err := index.Search(q, searchMaxScore, true)
if err != nil {
return nil, err
}
// Sort results.
search.SortScore(results)
return results, nil
return c.getIndexYaml(namespaces)
}
// getIndexYaml will get the index yaml files for all the namespaces and merge them
// as one unified index yaml file.
func (rh *RepositoryHandler) getIndexYaml(namespaces []string) (*helm_repo.IndexFile, error) {
func (c *Controller) getIndexYaml(namespaces []string) (*helm_repo.IndexFile, error) {
// The final merged index file
mergedIndexFile := &helm_repo.IndexFile{
APIVersion: "v1",
Entries: make(map[string]helm_repo.ChartVersions),
Generated: time.Now(),
Generated: time.Now().Round(time.Second),
PublicKeys: []string{},
}
@ -175,7 +79,7 @@ func (rh *RepositoryHandler) getIndexYaml(namespaces []string) (*helm_repo.Index
}()
for res := range resultChan {
rh.mergeIndexFile(res.namespace, mergedIndexFile, res.indexFileOfRepo)
c.mergeIndexFile(res.namespace, mergedIndexFile, res.indexFileOfRepo)
}
}()
@ -203,7 +107,7 @@ LOOP:
workerPool <- struct{}{}
}()
indexFile, err := rh.getIndexYamlWithNS(ns)
indexFile, err := c.getIndexYamlWithNS(ns)
if err != nil {
if len(errorChan) == 0 {
//Only need one error as failure signal
@ -254,13 +158,13 @@ LOOP:
}
// Get the index yaml file under the specified namespace from the backend server
func (rh *RepositoryHandler) getIndexYamlWithNS(namespace string) (*helm_repo.IndexFile, error) {
func (c *Controller) getIndexYamlWithNS(namespace string) (*helm_repo.IndexFile, error) {
// Join url path
url := path.Join(namespace, "index.yaml")
url = fmt.Sprintf("%s/%s", rh.backendServerAddress.String(), url)
url = fmt.Sprintf("%s/%s", c.backendServerAddress.String(), url)
hlog.Debugf("Getting index.yaml from '%s'", url)
content, err := rh.apiClient.GetContent(url)
content, err := c.apiClient.GetContent(url)
if err != nil {
return nil, err
}
@ -276,7 +180,7 @@ func (rh *RepositoryHandler) getIndexYamlWithNS(namespace string) (*helm_repo.In
// Merge the content of mergingIndexFile to the baseIndex
// The chart url should be without --chart-url prefix
func (rh *RepositoryHandler) mergeIndexFile(namespace string,
func (c *Controller) mergeIndexFile(namespace string,
baseIndex *helm_repo.IndexFile,
mergingIndexFile *helm_repo.IndexFile) {
// Append entries
@ -304,12 +208,9 @@ func (rh *RepositoryHandler) mergeIndexFile(namespace string,
}
// Generate empty index file
func emptyIndexFile() []byte {
func emptyIndexFile() *helm_repo.IndexFile {
emptyIndexFile := &helm_repo.IndexFile{}
emptyIndexFile.Generated = time.Now()
emptyIndexFile.Generated = time.Now().Round(time.Second)
// Ignore the error
rawData, _ := json.Marshal(emptyIndexFile)
return rawData
return emptyIndexFile
}

32
src/chartserver/handler_repo_test.go Normal file
View File

@ -0,0 +1,32 @@
package chartserver
import "testing"
// Test get /index.yaml
func TestGetIndexFile(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
namespaces := []string{"repo1", "repo2"}
indexFile, err := c.GetIndexFile(namespaces)
if err != nil {
t.Fatal(err)
}
if len(indexFile.Entries) != 5 {
t.Fatalf("Expect index file with 5 entries, but got %d", len(indexFile.Entries))
}
_, ok := indexFile.Entries["repo1/harbor"]
if !ok {
t.Fatal("Expect chart entry 'repo1/harbor' but got nothing")
}
_, ok = indexFile.Entries["repo2/harbor"]
if !ok {
t.Fatal("Expect chart entry 'repo2/harbor' but got nothing")
}
}

208
src/chartserver/handler_utility.go Normal file
View File

@ -0,0 +1,208 @@
package chartserver
import (
"errors"
"fmt"
"path"
"strings"
"sync"
hlog "github.com/goharbor/harbor/src/common/utils/log"
"k8s.io/helm/cmd/helm/search"
)
const (
maxDeletionThreads = 10
)
// DeleteChart deletes all the chart versions of the specified chart under the namespace.
// See @ServiceHandler.DeleteChart
func (c *Controller) DeleteChart(namespace, chartName string) error {
if len(strings.TrimSpace(namespace)) == 0 {
return errors.New("empty namespace when deleting chart")
}
if len(strings.TrimSpace(chartName)) == 0 {
return errors.New("empty chart name when deleting chart")
}
url := fmt.Sprintf("%s/%s", c.APIPrefix(namespace), chartName)
content, err := c.apiClient.GetContent(url)
if err != nil {
return err
}
allVersions, err := c.chartOperator.GetChartVersions(content)
if err != nil {
return err
}
// Let's delete the versions in parallel
// The number of goroutine is controlled by the const maxDeletionThreads
qSize := len(allVersions)
if qSize > maxDeletionThreads {
qSize = maxDeletionThreads
}
tokenQueue := make(chan struct{}, qSize)
errChan := make(chan error, 1)
waitGroup := new(sync.WaitGroup)
waitGroup.Add(len(allVersions))
// Append initial tokens
for i := 0; i < qSize; i++ {
tokenQueue <- struct{}{}
}
// Collect errors
errs := make([]error, 0)
errWrapper := make(chan error, 1)
go func() {
defer func() {
// pass to the out func
if len(errs) > 0 {
errWrapper <- fmt.Errorf("%v", errs)
}
close(errWrapper)
}()
for deletionErr := range errChan {
errs = append(errs, deletionErr)
}
}()
// Schedule deletion tasks
for _, deletingVersion := range allVersions {
// Apply for token first
// If no available token, pending here
<-tokenQueue
// Got one token
go func(deletingVersion *ChartVersion) {
defer func() {
// return the token back
tokenQueue <- struct{}{}
// done
waitGroup.Done()
}()
if err := c.DeleteChartVersion(namespace, chartName, deletingVersion.GetVersion()); err != nil {
errChan <- err
}
}(deletingVersion)
}
// Wait all goroutines are done
waitGroup.Wait()
// Safe to quit error collection goroutine
close(errChan)
err = <-errWrapper
return err
}
// GetChartVersionDetails get the specified version for one chart
// This handler should return the details of the chart version,
// maybe including metadata,dependencies and values etc.
// See @ServiceHandler.GetChartVersionDetails
func (c *Controller) GetChartVersionDetails(namespace, chartName, version string) (*ChartVersionDetails, error) {
chartV, err := c.GetChartVersion(namespace, chartName, version)
if err != nil {
return nil, err
}
// Query cache
chartDetails := c.chartCache.GetChart(chartV.Digest)
if chartDetails == nil {
// NOT hit!!
content, err := c.getChartVersionContent(namespace, chartV.URLs[0])
if err != nil {
return nil, err
}
// Process bytes and get more details of chart version
chartDetails, err = c.chartOperator.GetChartDetails(content)
if err != nil {
return nil, err
}
chartDetails.Metadata = chartV
// Put it into the cache for next access
c.chartCache.PutChart(chartDetails)
} else {
// Just logged
hlog.Debugf("Get detailed data from cache for chart: %s:%s (%s)",
chartDetails.Metadata.Name,
chartDetails.Metadata.Version,
chartDetails.Metadata.Digest)
}
// The change of prov file will not cause any influence to the digest of chart,
// and then the digital signature status should be not cached
//
// Generate the security report
// prov file share same endpoint with the chart version
// Just add .prov suffix to the chart version to form the path of prov file
// Anyway, there will be a report about the digital signature status
chartDetails.Security = &SecurityReport{
Signature: &DigitalSignature{
Signed: false,
},
}
// Try to get the prov file to confirm if it is exitsing
provFilePath := fmt.Sprintf("%s.prov", chartV.URLs[0])
provBytes, err := c.getChartVersionContent(namespace, provFilePath)
if err == nil && len(provBytes) > 0 {
chartDetails.Security.Signature.Signed = true
chartDetails.Security.Signature.Provenance = provFilePath
} else {
// Just log it
hlog.Errorf("Failed to get prov file for chart %s with error: %s, got %d bytes", chartV.Name, err.Error(), len(provBytes))
}
return chartDetails, nil
}
// SearchChart search charts in the specified namespaces with the keyword q.
// RegExp mode is enabled as default.
// For each chart, only the latest version will shown in the result list if matched to avoid duplicated entries.
// Keep consistent with `helm search` command.
func (c *Controller) SearchChart(q string, namespaces []string) ([]*search.Result, error) {
if len(q) == 0 || len(namespaces) == 0 {
// Return empty list
return []*search.Result{}, nil
}
// Get the merged index yaml file of the namespaces
ind, err := c.getIndexYaml(namespaces)
if err != nil {
return nil, err
}
// Build the search index
index := search.NewIndex()
// As the repo name is already merged into the index yaml, we use empty repo name.
// Set 'All' to false to return only one version for each chart.
index.AddRepo("", ind, false)
// Search
// RegExp is enabled
results, err := index.Search(q, searchMaxScore, true)
if err != nil {
return nil, err
}
// Sort results.
search.SortScore(results)
return results, nil
}
// Get the content bytes of the chart version
func (c *Controller) getChartVersionContent(namespace string, subPath string) ([]byte, error) {
url := path.Join(namespace, subPath)
url = fmt.Sprintf("%s/%s", c.backendServerAddress.String(), url)
return c.apiClient.GetContent(url)
}

83
src/chartserver/handler_utility_test.go Normal file
View File

@ -0,0 +1,83 @@
package chartserver
import (
"testing"
)
// Test the function DeleteChart
func TestDeleteChart(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
if err := c.DeleteChart("repo1", "harbor"); err != nil {
t.Fatal(err)
}
}
// Test get /api/:repo/charts/:chart_name/:version
func TestGetChartVersion(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
chartVersion, err := c.GetChartVersionDetails("repo1", "harbor", "0.2.0")
if err != nil {
t.Fatal(err)
}
if chartVersion.Metadata.Name != "harbor" {
t.Fatalf("Expect harbor chart version but got %s", chartVersion.Metadata.Name)
}
if chartVersion.Metadata.Version != "0.2.0" {
t.Fatalf("Expect version '0.2.0' but got version %s", chartVersion.Metadata.Version)
}
if len(chartVersion.Dependencies) != 1 {
t.Fatalf("Expect 1 dependency but got %d ones", len(chartVersion.Dependencies))
}
if len(chartVersion.Values) != 99 {
t.Fatalf("Expect 99 k-v values but got %d", len(chartVersion.Values))
}
}
// Test get /api/:repo/charts/:chart_name/:version with none-existing version
func TestGetChartVersionWithError(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
_, err = c.GetChartVersionDetails("repo1", "harbor", "1.0.0")
if err == nil {
t.Fatal("Expect an error but got nil")
}
}
// Test the chart searching
func TestChartSearching(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
namespaces := []string{"repo1", "repo2"}
q := "harbor"
results, err := c.SearchChart(q, namespaces)
if err != nil {
t.Fatalf("expect nil error but got '%s'", err)
}
if len(results) != 2 {
t.Fatalf("expect 2 results but got %d", len(results))
}
}

199
src/chartserver/manipulation_handler.go
View File

@ -1,199 +0,0 @@
package chartserver
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"path"
"strings"
"github.com/ghodss/yaml"
hlog "github.com/goharbor/harbor/src/common/utils/log"
helm_repo "k8s.io/helm/pkg/repo"
)
const (
// NamespaceContextKey is context key for the namespace
NamespaceContextKey ContextKey = ":repo"
)
// ContextKey is defined for add value in the context of http request
type ContextKey string
// ManipulationHandler includes all the handler methods for the purpose of manipulating the
// chart repository
type ManipulationHandler struct {
// Proxy used to to transfer the traffic of requests
// It's mainly used to talk to the backend chart server
trafficProxy *ProxyEngine
// Parse and process the chart version to provide required info data
chartOperator *ChartOperator
// HTTP client used to call the realted APIs of the backend chart repositories
apiClient *ChartClient
// Point to the url of the backend server
backendServerAddress *url.URL
// Cache the chart data
chartCache *ChartCache
}
// ListCharts lists all the charts under the specified namespace
func (mh *ManipulationHandler) ListCharts(w http.ResponseWriter, req *http.Request) {
url := strings.TrimPrefix(req.URL.String(), "/")
url = fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), url)
content, err := mh.apiClient.GetContent(url)
if err != nil {
WriteInternalError(w, err)
return
}
chartList, err := mh.chartOperator.GetChartList(content)
if err != nil {
WriteInternalError(w, err)
return
}
jsonData, err := json.Marshal(chartList)
if err != nil {
WriteInternalError(w, err)
return
}
writeJSONData(w, jsonData)
}
// GetChart returns all the chart versions under the specified chart
func (mh *ManipulationHandler) GetChart(w http.ResponseWriter, req *http.Request) {
mh.trafficProxy.ServeHTTP(w, req)
}
// GetChartVersion get the specified version for one chart
// This handler should return the details of the chart version,
// maybe including metadata,dependencies and values etc.
func (mh *ManipulationHandler) GetChartVersion(w http.ResponseWriter, req *http.Request) {
chartV, err := mh.getChartVersion(req.URL.String())
if err != nil {
WriteInternalError(w, err)
return
}
// Get and check namespace
// even we get the data from cache
var namespace string
repoValue := req.Context().Value(NamespaceContextKey)
if repoValue != nil {
if ns, ok := repoValue.(string); ok {
namespace = ns
}
}
if len(strings.TrimSpace(namespace)) == 0 {
WriteInternalError(w, errors.New("failed to extract namespace from the request"))
return
}
// Query cache
chartDetails := mh.chartCache.GetChart(chartV.Digest)
if chartDetails == nil {
// NOT hit!!
content, err := mh.getChartVersionContent(namespace, chartV.URLs[0])
if err != nil {
WriteInternalError(w, err)
return
}
// Process bytes and get more details of chart version
chartDetails, err = mh.chartOperator.GetChartDetails(content)
if err != nil {
WriteInternalError(w, err)
return
}
chartDetails.Metadata = chartV
// Put it into the cache for next access
mh.chartCache.PutChart(chartDetails)
} else {
// Just logged
hlog.Debugf("Get detailed data from cache for chart: %s:%s (%s)",
chartDetails.Metadata.Name,
chartDetails.Metadata.Version,
chartDetails.Metadata.Digest)
}
// The change of prov file will not cause any influence to the digest of chart,
// and then the digital signature status should be not cached
//
// Generate the security report
// prov file share same endpoint with the chart version
// Just add .prov suffix to the chart version to form the path of prov file
// Anyway, there will be a report about the digital signature status
chartDetails.Security = &SecurityReport{
Signature: &DigitalSignature{
Signed: false,
},
}
// Try to get the prov file to confirm if it is exitsing
provFilePath := fmt.Sprintf("%s.prov", chartV.URLs[0])
provBytes, err := mh.getChartVersionContent(namespace, provFilePath)
if err == nil && len(provBytes) > 0 {
chartDetails.Security.Signature.Signed = true
chartDetails.Security.Signature.Provenance = provFilePath
} else {
// Just log it
hlog.Errorf("Failed to get prov file for chart %s with error: %s, got %d bytes", chartV.Name, err.Error(), len(provBytes))
}
bytes, err := json.Marshal(chartDetails)
if err != nil {
WriteInternalError(w, err)
return
}
writeJSONData(w, bytes)
}
// UploadChartVersion will save the new version of the chart to the backend storage
func (mh *ManipulationHandler) UploadChartVersion(w http.ResponseWriter, req *http.Request) {
mh.trafficProxy.ServeHTTP(w, req)
}
// UploadProvenanceFile will save the provenance file of the chart to the backend storage
func (mh *ManipulationHandler) UploadProvenanceFile(w http.ResponseWriter, req *http.Request) {
mh.trafficProxy.ServeHTTP(w, req)
}
// DeleteChartVersion will delete the specified version of the chart
func (mh *ManipulationHandler) DeleteChartVersion(w http.ResponseWriter, req *http.Request) {
mh.trafficProxy.ServeHTTP(w, req)
}
// Get the basic metadata of chart version
func (mh *ManipulationHandler) getChartVersion(subPath string) (*helm_repo.ChartVersion, error) {
url := fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), strings.TrimPrefix(subPath, "/"))
content, err := mh.apiClient.GetContent(url)
if err != nil {
return nil, err
}
chartVersion := &helm_repo.ChartVersion{}
if err := yaml.Unmarshal(content, chartVersion); err != nil {
return nil, err
}
return chartVersion, nil
}
// Get the content bytes of the chart version
func (mh *ManipulationHandler) getChartVersionContent(namespace string, subPath string) ([]byte, error) {
url := path.Join(namespace, subPath)
url = fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), url)
return mh.apiClient.GetContent(url)
}

37
src/chartserver/reverse_proxy.go
View File

@ -17,6 +17,11 @@ import (
const (
agentHarbor = "HARBOR"
contentLengthHeader = "Content-Length"
defaultRepo = "library"
rootUploadingEndpoint = "/api/chartrepo/charts"
rootIndexEndpoint = "/chartrepo/index.yaml"
chartRepoHealthEndpoint = "/api/chartrepo/health"
)
// ProxyEngine is used to proxy the related traffics
@ -56,6 +61,7 @@ func director(target *url.URL, cred *Credential, req *http.Request) {
// Overwrite the request URL to the target path
req.URL.Scheme = target.Scheme
req.URL.Host = target.Host
rewriteURLPath(req)
req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path)
if targetQuery == "" || req.URL.RawQuery == "" {
req.URL.RawQuery = targetQuery + req.URL.RawQuery
@ -125,3 +131,34 @@ func singleJoiningSlash(a, b string) string {
}
return a + b
}
// Rewrite the incoming URL with the right backend URL pattern
// Remove 'chartrepo' from the endpoints of manipulation API
// Remove 'chartrepo' from the endpoints of repository services
func rewriteURLPath(req *http.Request) {
incomingURLPath := req.RequestURI
// Health check endpoint
if incomingURLPath == chartRepoHealthEndpoint {
req.URL.Path = "/health"
return
}
// Root uploading endpoint
if incomingURLPath == rootUploadingEndpoint {
req.URL.Path = strings.Replace(incomingURLPath, "chartrepo", defaultRepo, 1)
return
}
// Repository endpoints
if strings.HasPrefix(incomingURLPath, "/chartrepo") {
req.URL.Path = strings.TrimPrefix(incomingURLPath, "/chartrepo")
return
}
// API endpoints
if strings.HasPrefix(incomingURLPath, "/api/chartrepo") {
req.URL.Path = strings.Replace(incomingURLPath, "/chartrepo", "", 1)
return
}
}

163
src/chartserver/utility_handler.go
View File

@ -1,163 +0,0 @@
package chartserver
import (
"errors"
"fmt"
"net/url"
"strings"
"sync"
"github.com/ghodss/yaml"
helm_repo "k8s.io/helm/pkg/repo"
)
const (
maxDeletionThreads = 10
)
// UtilityHandler provides utility methods
type UtilityHandler struct {
// Parse and process the chart version to provide required info data
chartOperator *ChartOperator
// HTTP client used to call the realted APIs of the backend chart repositories
apiClient *ChartClient
// Point to the url of the backend server
backendServerAddress *url.URL
}
// GetChartsByNs gets the chart list under the namespace
func (uh *UtilityHandler) GetChartsByNs(namespace string) ([]*ChartInfo, error) {
if len(strings.TrimSpace(namespace)) == 0 {
return nil, errors.New("empty namespace when getting chart list")
}
path := fmt.Sprintf("/api/%s/charts", namespace)
url := fmt.Sprintf("%s%s", uh.backendServerAddress.String(), path)
content, err := uh.apiClient.GetContent(url)
if err != nil {
return nil, err
}
return uh.chartOperator.GetChartList(content)
}
// DeleteChart deletes all the chart versions of the specified chart under the namespace.
func (uh *UtilityHandler) DeleteChart(namespace, chartName string) error {
if len(strings.TrimSpace(namespace)) == 0 {
return errors.New("empty namespace when deleting chart")
}
if len(strings.TrimSpace(chartName)) == 0 {
return errors.New("empty chart name when deleting chart")
}
path := fmt.Sprintf("/api/%s/charts/%s", namespace, chartName)
url := fmt.Sprintf("%s%s", uh.backendServerAddress.String(), path)
content, err := uh.apiClient.GetContent(url)
if err != nil {
return err
}
allVersions, err := uh.chartOperator.GetChartVersions(content)
if err != nil {
return err
}
// Let's delete the versions in parallel
// The number of goroutine is controlled by the const maxDeletionThreads
qSize := len(allVersions)
if qSize > maxDeletionThreads {
qSize = maxDeletionThreads
}
tokenQueue := make(chan struct{}, qSize)
errChan := make(chan error, 1)
waitGroup := new(sync.WaitGroup)
waitGroup.Add(len(allVersions))
// Append initial tokens
for i := 0; i < qSize; i++ {
tokenQueue <- struct{}{}
}
// Collect errors
errs := make([]error, 0)
errWrapper := make(chan error, 1)
go func() {
defer func() {
// pass to the out func
if len(errs) > 0 {
errWrapper <- fmt.Errorf("%v", errs)
}
close(errWrapper)
}()
for deletionErr := range errChan {
errs = append(errs, deletionErr)
}
}()
// Schedule deletion tasks
for _, deletingVersion := range allVersions {
// Apply for token first
// If no available token, pending here
<-tokenQueue
// Got one token
go func(deletingVersion *helm_repo.ChartVersion) {
defer func() {
// return the token back
tokenQueue <- struct{}{}
// done
waitGroup.Done()
}()
if err := uh.deleteChartVersion(namespace, chartName, deletingVersion.GetVersion()); err != nil {
errChan <- err
}
}(deletingVersion)
}
// Wait all goroutines are done
waitGroup.Wait()
// Safe to quit error collection goroutine
close(errChan)
err = <-errWrapper
return err
}
// GetChartVersion returns the summary of the specified chart version.
func (uh *UtilityHandler) GetChartVersion(namespace, name, version string) (*helm_repo.ChartVersion, error) {
if len(namespace) == 0 || len(name) == 0 || len(version) == 0 {
return nil, errors.New("bad arguments to get chart version summary")
}
path := fmt.Sprintf("/api/%s/charts/%s/%s", namespace, name, version)
url := fmt.Sprintf("%s%s", uh.backendServerAddress.String(), path)
content, err := uh.apiClient.GetContent(url)
if err != nil {
return nil, err
}
chartVersion := &helm_repo.ChartVersion{}
if err := yaml.Unmarshal(content, chartVersion); err != nil {
return nil, err
}
return chartVersion, nil
}
// deleteChartVersion deletes the specified chart version
func (uh *UtilityHandler) deleteChartVersion(namespace, chartName, version string) error {
path := fmt.Sprintf("/api/%s/charts/%s/%s", namespace, chartName, version)
url := fmt.Sprintf("%s%s", uh.backendServerAddress.String(), path)
return uh.apiClient.DeleteContent(url)
}

228
src/chartserver/utility_handler_test.go
View File

@ -1,228 +0,0 @@
package chartserver
import (
"net/http"
"net/http/httptest"
"net/url"
"testing"
)
// TestGetChartsByNs tests GetChartsByNs method in UtilityHandler
func TestGetChartsByNs(t *testing.T) {
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.RequestURI {
case "/api/repo1/charts":
if r.Method == http.MethodGet {
w.Write(chartListContent)
return
}
}
w.WriteHeader(http.StatusNotImplemented)
w.Write([]byte("not supported"))
}))
defer mockServer.Close()
serverURL, err := url.Parse(mockServer.URL)
if err != nil {
t.Fatal(err)
}
theController, err := NewController(serverURL)
if err != nil {
t.Fatal(err)
}
charts, err := theController.GetUtilityHandler().GetChartsByNs("repo1")
if err != nil {
t.Fatal(err)
}
if len(charts) != 2 {
t.Fatalf("expect 2 items but got %d", len(charts))
}
}
// Test the function DeleteChart
func TestDeleteChart(t *testing.T) {
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.RequestURI {
case "/api/repo1/charts/harbor":
if r.Method == http.MethodGet {
w.Write([]byte(chartVersionsOfHarbor))
return
}
case "/api/repo1/charts/harbor/0.2.0",
"/api/repo1/charts/harbor/0.2.1":
if r.Method == http.MethodDelete {
w.WriteHeader(http.StatusOK)
return
}
}
w.WriteHeader(http.StatusNotImplemented)
w.Write([]byte("not supported"))
}))
defer mockServer.Close()
serverURL, err := url.Parse(mockServer.URL)
if err != nil {
t.Fatal(err)
}
theController, err := NewController(serverURL)
if err != nil {
t.Fatal(err)
}
if err := theController.GetUtilityHandler().DeleteChart("repo1", "harbor"); err != nil {
t.Fatal(err)
}
}
// Test the GetChartVersion in utility handler
func TestGetChartVersionSummary(t *testing.T) {
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.RequestURI {
case "/api/repo1/charts/harbor/0.2.0":
if r.Method == http.MethodGet {
w.Write([]byte(chartVersionOfHarbor020))
return
}
}
w.WriteHeader(http.StatusNotImplemented)
w.Write([]byte("not supported"))
}))
defer mockServer.Close()
serverURL, err := url.Parse(mockServer.URL)
if err != nil {
t.Fatal(err)
}
theController, err := NewController(serverURL)
if err != nil {
t.Fatal(err)
}
chartV, err := theController.GetUtilityHandler().GetChartVersion("repo1", "harbor", "0.2.0")
if err != nil {
t.Fatal(err)
}
if chartV.GetName() != "harbor" {
t.Fatalf("expect chart name 'harbor' but got '%s'", chartV.GetName())
}
if chartV.GetVersion() != "0.2.0" {
t.Fatalf("expect chart version '0.2.0' but got '%s'", chartV.GetVersion())
}
}
var chartVersionOfHarbor020 = `
{
"name": "harbor",
"home": "https://github.com/vmware/harbor",
"sources": [
"https://github.com/vmware/harbor/tree/master/contrib/helm/harbor"
],
"version": "0.2.0",
"description": "An Enterprise-class Docker Registry by VMware",
"keywords": [
"vmware",
"docker",
"registry",
"harbor"
],
"maintainers": [
{
"name": "Jesse Hu",
"email": "huh@vmware.com"
},
{
"name": "paulczar",
"email": "username.taken@gmail.com"
}
],
"engine": "gotpl",
"icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png",
"appVersion": "1.5.0",
"urls": [
"charts/harbor-0.2.0.tgz"
],
"created": "2018-08-29T10:26:21.141611102Z",
"digest": "fc8aae8dade9f0dfca12e9f1085081c49843d30a063a3fa7eb42497e3ceb277c"
}
`
var chartVersionsOfHarbor = `
[
{
"name": "harbor",
"home": "https://github.com/vmware/harbor",
"sources": [
"https://github.com/vmware/harbor/tree/master/contrib/helm/harbor"
],
"version": "0.2.1",
"description": "An Enterprise-class Docker Registry by VMware",
"keywords": [
"vmware",
"docker",
"registry",
"harbor"
],
"maintainers": [
{
"name": "Jesse Hu",
"email": "huh@vmware.com"
},
{
"name": "paulczar",
"email": "username.taken@gmail.com"
}
],
"engine": "gotpl",
"icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png",
"appVersion": "1.5.0",
"urls": [
"charts/harbor-0.2.1.tgz"
],
"created": "2018-08-29T10:26:29.625749155Z",
"digest": "2538edf4ddb797af8e025f3bd6226270440110bbdb689bad48656a519a154236"
},
{
"name": "harbor",
"home": "https://github.com/vmware/harbor",
"sources": [
"https://github.com/vmware/harbor/tree/master/contrib/helm/harbor"
],
"version": "0.2.0",
"description": "An Enterprise-class Docker Registry by VMware",
"keywords": [
"vmware",
"docker",
"registry",
"harbor"
],
"maintainers": [
{
"name": "Jesse Hu",
"email": "huh@vmware.com"
},
{
"name": "paulczar",
"email": "username.taken@gmail.com"
}
],
"engine": "gotpl",
"icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png",
"appVersion": "1.5.0",
"urls": [
"charts/harbor-0.2.0.tgz"
],
"created": "2018-08-29T10:26:21.141611102Z",
"digest": "fc8aae8dade9f0dfca12e9f1085081c49843d30a063a3fa7eb42497e3ceb277c"
}
]
`

61
src/chartserver/utils.go
View File

@ -4,8 +4,8 @@ import (
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"os"
"strings"
)
@ -14,30 +14,6 @@ const (
contentTypeJSON = "application/json"
)
// WriteError writes error to http client
func WriteError(w http.ResponseWriter, code int, err error) {
errorObj := make(map[string]string)
errorObj["error"] = err.Error()
errorContent, errorMarshal := json.Marshal(errorObj)
if errorMarshal != nil {
errorContent = []byte(err.Error())
}
w.WriteHeader(code)
w.Write(errorContent)
}
// WriteInternalError writes error with statusCode == 500
func WriteInternalError(w http.ResponseWriter, err error) {
WriteError(w, http.StatusInternalServerError, err)
}
// Write JSON data to http client
func writeJSONData(w http.ResponseWriter, data []byte) {
w.Header().Set(contentTypeHeader, contentTypeJSON)
w.WriteHeader(http.StatusOK)
w.Write(data)
}
// Extract error object '{"error": "****---***"}' from the content if existing
// nil error will be returned if it does exist
func extractError(content []byte) error {
@ -116,3 +92,38 @@ func parseRedisConfig(redisConfigV string) (string, error) {
return string(cfgData), nil
}
// What's the cache driver if it is set
func parseCacheDriver() (string, bool) {
driver, ok := os.LookupEnv(cacheDriverENVKey)
return strings.ToLower(driver), ok
}
// Get and parse the configuration for the chart cache
func getCacheConfig() (*ChartCacheConfig, error) {
driver, isSet := parseCacheDriver()
if !isSet {
return nil, nil
}
if driver != cacheDriverMem && driver != cacheDriverRedis {
return nil, fmt.Errorf("cache driver '%s' is not supported, only support 'memory' and 'redis'", driver)
}
if driver == cacheDriverMem {
return &ChartCacheConfig{
DriverType: driver,
}, nil
}
redisConfigV := os.Getenv(redisENVKey)
redisCfg, err := parseRedisConfig(redisConfigV)
if err != nil {
return nil, fmt.Errorf("failed to parse redis configurations from '%s' with error: %s", redisCfg, err)
}
return &ChartCacheConfig{
DriverType: driver,
Config: redisCfg,
}, nil
}

25
src/ui/api/base.go
View File

@ -17,6 +17,7 @@ package api
import (
"net/http"
yaml "github.com/ghodss/yaml"
"github.com/goharbor/harbor/src/common/api"
"github.com/goharbor/harbor/src/common/security"
"github.com/goharbor/harbor/src/common/utils/log"
@ -26,6 +27,10 @@ import (
"github.com/goharbor/harbor/src/ui/utils"
)
const (
yamlFileContentType = "application/x-yaml"
)
// BaseController ...
type BaseController struct {
api.BaseAPI
@ -98,6 +103,26 @@ func (b *BaseController) SendForbiddenError(err error) {
b.RenderFormatedError(http.StatusForbidden, err)
}
// WriteJSONData writes the JSON data to the client.
func (b *BaseController) WriteJSONData(object interface{}) {
b.Data["json"] = object
b.ServeJSON()
}
// WriteYamlData writes the yaml data to the client.
func (b *BaseController) WriteYamlData(object interface{}) {
yData, err := yaml.Marshal(object)
if err != nil {
b.SendInternalServerError(err)
return
}
w := b.Ctx.ResponseWriter
w.Header().Set("Content-Type", yamlFileContentType)
w.WriteHeader(http.StatusOK)
w.Write(yData)
}
// Init related objects/configurations for the API controllers
func Init() error {
// If chart repository is not enabled then directly return

2
src/ui/api/chart_label.go
View File

@ -55,7 +55,7 @@ func (cla *ChartLabelAPI) Prepare() {
chartName := cla.GetStringFromPath(nameParam)
version := cla.GetStringFromPath(versionParam)
if _, err = chartController.GetUtilityHandler().GetChartVersion(project, chartName, version); err != nil {
if _, err = chartController.GetChartVersion(project, chartName, version); err != nil {
cla.SendNotFoundError(err)
return
}

199
src/ui/api/chart_repository.go
View File

@ -2,7 +2,6 @@ package api
import (
"bytes"
"context"
"errors"
"fmt"
"io"
@ -12,6 +11,9 @@ import (
"net/url"
"strings"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/ui/label"
"github.com/goharbor/harbor/src/chartserver"
hlog "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/ui/config"
@ -45,6 +47,9 @@ type ChartRepositoryAPI struct {
// The base controller to provide common utilities
BaseController
// For label management
labelManager *label.BaseManager
// Keep the namespace if existing
namespace string
}
@ -75,18 +80,19 @@ func (cra *ChartRepositoryAPI) Prepare() {
}
}
// Rewrite URL path
cra.rewriteURLPath(cra.Ctx.Request)
// Init label manager
cra.labelManager = &label.BaseManager{}
}
// GetHealthStatus handles GET /api/chartserver/health
// GetHealthStatus handles GET /api/chartrepo/health
func (cra *ChartRepositoryAPI) GetHealthStatus() {
// Check access
if !cra.requireAccess(cra.namespace, accessLevelSystem) {
return
}
chartController.GetBaseHandler().GetHealthStatus(cra.Ctx.ResponseWriter, cra.Ctx.Request)
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// GetIndexByRepo handles GET /:repo/index.yaml
@ -96,7 +102,8 @@ func (cra *ChartRepositoryAPI) GetIndexByRepo() {
return
}
chartController.GetRepositoryHandler().GetIndexFileWithNS(cra.Ctx.ResponseWriter, cra.Ctx.Request)
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// GetIndex handles GET /index.yaml
@ -106,7 +113,24 @@ func (cra *ChartRepositoryAPI) GetIndex() {
return
}
chartController.GetRepositoryHandler().GetIndexFile(cra.Ctx.ResponseWriter, cra.Ctx.Request)
results, err := cra.ProjectMgr.List(nil)
if err != nil {
cra.SendInternalServerError(err)
return
}
namespaces := []string{}
for _, r := range results.Projects {
namespaces = append(namespaces, r.Name)
}
indexFile, err := chartController.GetIndexFile(namespaces)
if err != nil {
cra.SendInternalServerError(err)
return
}
cra.WriteYamlData(indexFile)
}
// DownloadChart handles GET /:repo/charts/:filename
@ -116,7 +140,8 @@ func (cra *ChartRepositoryAPI) DownloadChart() {
return
}
chartController.GetRepositoryHandler().DownloadChartObject(cra.Ctx.ResponseWriter, cra.Ctx.Request)
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// ListCharts handles GET /api/:repo/charts
@ -126,7 +151,13 @@ func (cra *ChartRepositoryAPI) ListCharts() {
return
}
chartController.GetManipulationHandler().ListCharts(cra.Ctx.ResponseWriter, cra.Ctx.Request)
charts, err := chartController.ListCharts(cra.namespace)
if err != nil {
cra.SendInternalServerError(err)
return
}
cra.WriteJSONData(charts)
}
// ListChartVersions GET /api/:repo/charts/:name
@ -136,7 +167,25 @@ func (cra *ChartRepositoryAPI) ListChartVersions() {
return
}
chartController.GetManipulationHandler().GetChart(cra.Ctx.ResponseWriter, cra.Ctx.Request)
chartName := cra.GetStringFromPath(nameParam)
versions, err := chartController.GetChart(cra.namespace, chartName)
if err != nil {
cra.SendInternalServerError(err)
return
}
// Append labels
for _, chartVersion := range versions {
labels, err := cra.labelManager.GetLabelsOfResource(common.ResourceTypeChart, chartFullName(cra.namespace, chartVersion.Name, chartVersion.Version))
if err != nil {
cra.SendInternalServerError(err)
return
}
chartVersion.Labels = labels
}
cra.WriteJSONData(versions)
}
// GetChartVersion handles GET /api/:repo/charts/:name/:version
@ -146,11 +195,25 @@ func (cra *ChartRepositoryAPI) GetChartVersion() {
return
}
// Let's pass the namespace via the context of request
req := cra.Ctx.Request
*req = *(req.WithContext(context.WithValue(req.Context(), chartserver.NamespaceContextKey, cra.namespace)))
// Get other parameters
chartName := cra.GetStringFromPath(nameParam)
version := cra.GetStringFromPath(versionParam)
chartController.GetManipulationHandler().GetChartVersion(cra.Ctx.ResponseWriter, req)
chartVersion, err := chartController.GetChartVersionDetails(cra.namespace, chartName, version)
if err != nil {
cra.SendInternalServerError(err)
return
}
// Append labels
labels, err := cra.labelManager.GetLabelsOfResource(common.ResourceTypeChart, chartFullName(cra.namespace, chartName, version))
if err != nil {
cra.SendInternalServerError(err)
return
}
chartVersion.Labels = labels
cra.WriteJSONData(chartVersion)
}
// DeleteChartVersion handles DELETE /api/:repo/charts/:name/:version
@ -160,7 +223,20 @@ func (cra *ChartRepositoryAPI) DeleteChartVersion() {
return
}
chartController.GetManipulationHandler().DeleteChartVersion(cra.Ctx.ResponseWriter, cra.Ctx.Request)
// Get other parameters
chartName := cra.GetStringFromPath(nameParam)
version := cra.GetStringFromPath(versionParam)
// Try to remove labels from deleting chart if exitsing
if err := cra.removeLabelsFromChart(chartName, version); err != nil {
cra.SendInternalServerError(err)
return
}
if err := chartController.DeleteChartVersion(cra.namespace, chartName, version); err != nil {
cra.SendInternalServerError(err)
return
}
}
// UploadChartVersion handles POST /api/:repo/charts
@ -184,12 +260,13 @@ func (cra *ChartRepositoryAPI) UploadChartVersion() {
formField: formFiledNameForProv,
})
if err := cra.rewriteFileContent(formFiles, cra.Ctx.Request); err != nil {
chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err)
cra.SendInternalServerError(err)
return
}
}
chartController.GetManipulationHandler().UploadChartVersion(cra.Ctx.ResponseWriter, cra.Ctx.Request)
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// UploadChartProvFile handles POST /api/:repo/prov
@ -208,12 +285,13 @@ func (cra *ChartRepositoryAPI) UploadChartProvFile() {
mustHave: true,
})
if err := cra.rewriteFileContent(formFiles, cra.Ctx.Request); err != nil {
chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err)
cra.SendInternalServerError(err)
return
}
}
chartController.GetManipulationHandler().UploadProvenanceFile(cra.Ctx.ResponseWriter, cra.Ctx.Request)
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// DeleteChart deletes all the chart versions of the specified chart.
@ -226,44 +304,39 @@ func (cra *ChartRepositoryAPI) DeleteChart() {
// Get other parameters from the request
chartName := cra.GetStringFromPath(nameParam)
if err := chartController.GetUtilityHandler().DeleteChart(cra.namespace, chartName); err != nil {
chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err)
// Remove labels from all the deleting chart versions under the chart
chartVersions, err := chartController.GetChart(cra.namespace, chartName)
if err != nil {
cra.SendInternalServerError(err)
return
}
for _, chartVersion := range chartVersions {
if err := cra.removeLabelsFromChart(chartName, chartVersion.GetVersion()); err != nil {
cra.SendInternalServerError(err)
return
}
}
if err := chartController.DeleteChart(cra.namespace, chartName); err != nil {
cra.SendInternalServerError(err)
return
}
}
// Rewrite the incoming URL with the right backend URL pattern
// Remove 'chartrepo' from the endpoints of manipulation API
// Remove 'chartrepo' from the endpoints of repository services
func (cra *ChartRepositoryAPI) rewriteURLPath(req *http.Request) {
incomingURLPath := req.RequestURI
defer func() {
hlog.Debugf("Incoming URL '%s' is rewritten to '%s'", incomingURLPath, req.URL.String())
}()
// Health check endpoint
if incomingURLPath == chartRepoHealthEndpoint {
req.URL.Path = "/health"
return
func (cra *ChartRepositoryAPI) removeLabelsFromChart(chartName, version string) error {
// Try to remove labels from deleting chart if exitsing
resourceID := chartFullName(cra.namespace, chartName, version)
labels, err := cra.labelManager.GetLabelsOfResource(common.ResourceTypeChart, resourceID)
if err == nil && len(labels) > 0 {
for _, l := range labels {
if err := cra.labelManager.RemoveLabelFromResource(common.ResourceTypeChart, resourceID, l.ID); err != nil {
return err
}
}
}
// Root uploading endpoint
if incomingURLPath == rootUploadingEndpoint {
req.URL.Path = strings.Replace(incomingURLPath, "chartrepo", defaultRepo, 1)
return
}
// Repository endpoints
if strings.HasPrefix(incomingURLPath, "/chartrepo") {
req.URL.Path = strings.TrimPrefix(incomingURLPath, "/chartrepo")
return
}
// API endpoints
if strings.HasPrefix(incomingURLPath, "/api/chartrepo") {
req.URL.Path = strings.Replace(incomingURLPath, "/chartrepo", "", 1)
return
}
return nil
}
// Check if there exists a valid namespace
@ -272,20 +345,20 @@ func (cra *ChartRepositoryAPI) rewriteURLPath(req *http.Request) {
func (cra *ChartRepositoryAPI) requireNamespace(namespace string) bool {
// Actually, never should be like this
if len(namespace) == 0 {
cra.HandleBadRequest(":repo should be in the request URL")
cra.SendBadRequestError(errors.New(":repo should be in the request URL"))
return false
}
existsing, err := cra.ProjectMgr.Exists(namespace)
if err != nil {
// Check failed with error
cra.renderError(http.StatusInternalServerError, fmt.Sprintf("failed to check existence of namespace %s with error: %s", namespace, err.Error()))
cra.SendInternalServerError(fmt.Errorf("failed to check existence of namespace %s with error: %s", namespace, err.Error()))
return false
}
// Not existing
if !existsing {
cra.renderError(http.StatusBadRequest, fmt.Sprintf("namespace %s is not existing", namespace))
cra.SendBadRequestError(fmt.Errorf("namespace %s is not existing", namespace))
return false
}
@ -328,7 +401,7 @@ func (cra *ChartRepositoryAPI) requireAccess(namespace string, accessLevel uint)
}
default:
// access rejected for invalid scope
cra.renderError(http.StatusForbidden, "unrecognized access scope")
cra.SendForbiddenError(errors.New("unrecognized access scope"))
return false
}
@ -336,23 +409,18 @@ func (cra *ChartRepositoryAPI) requireAccess(namespace string, accessLevel uint)
if err != nil {
// Unauthenticated, return 401
if !cra.SecurityCtx.IsAuthenticated() {
cra.renderError(http.StatusUnauthorized, "Unauthorized")
cra.SendUnAuthorizedError(errors.New("Unauthorized"))
return false
}
// Authenticated, return 403
cra.renderError(http.StatusForbidden, err.Error())
cra.SendForbiddenError(err)
return false
}
return true
}
// write error message with unified format
func (cra *ChartRepositoryAPI) renderError(code int, text string) {
chartserver.WriteError(cra.Ctx.ResponseWriter, code, errors.New(text))
}
// formFile is used to represent the uploaded files in the form
type formFile struct {
// form field key contains the form file
@ -442,3 +510,8 @@ func initializeChartController() (*chartserver.Controller, error) {
func isMultipartFormData(req *http.Request) bool {
return strings.Contains(req.Header.Get(headerContentType), contentTypeMultipart)
}
// Return the chart full name
func chartFullName(namespace, chartName, version string) string {
return fmt.Sprintf("%s/%s:%s", namespace, chartName, version)
}

40
src/ui/api/chart_repository_test.go
View File

@ -9,46 +9,6 @@ import (
"github.com/goharbor/harbor/src/ui/promgr/metamgr"
)
// Test the URL rewrite function
func TestURLRewrite(t *testing.T) {
chartAPI := &ChartRepositoryAPI{}
req, err := createRequest(http.MethodGet, "/api/chartrepo/health")
if err != nil {
t.Fatal(err)
}
chartAPI.rewriteURLPath(req)
if req.URL.Path != "/health" {
t.Fatalf("Expect url format %s but got %s", "/health", req.URL.Path)
}
req, err = createRequest(http.MethodGet, "/api/chartrepo/library/charts")
if err != nil {
t.Fatal(err)
}
chartAPI.rewriteURLPath(req)
if req.URL.Path != "/api/library/charts" {
t.Fatalf("Expect url format %s but got %s", "/api/library/charts", req.URL.Path)
}
req, err = createRequest(http.MethodPost, "/api/chartrepo/charts")
if err != nil {
t.Fatal(err)
}
chartAPI.rewriteURLPath(req)
if req.URL.Path != "/api/library/charts" {
t.Fatalf("Expect url format %s but got %s", "/api/library/charts", req.URL.Path)
}
req, err = createRequest(http.MethodGet, "/chartrepo/library/index.yaml")
if err != nil {
t.Fatal(err)
}
chartAPI.rewriteURLPath(req)
if req.URL.Path != "/library/index.yaml" {
t.Fatalf("Expect url format %s but got %s", "/library/index.yaml", req.URL.Path)
}
}
// Test access checking
func TestRequireAccess(t *testing.T) {
chartAPI := &ChartRepositoryAPI{}

2
src/ui/api/project.go
View File

@ -298,7 +298,7 @@ func (p *ProjectAPI) deletable(projectID int64) (*deletableResp, error) {
// Check helm charts number
if config.WithChartMuseum() {
charts, err := chartController.GetUtilityHandler().GetChartsByNs(p.project.Name)
charts, err := chartController.ListCharts(p.project.Name)
if err != nil {
return nil, err
}

2
src/ui/api/search.go
View File

@ -125,7 +125,7 @@ func (s *SearchAPI) Get() {
}
if searchHandler == nil {
searchHandler = chartController.GetRepositoryHandler().SearchChart
searchHandler = chartController.SearchChart
}
chartResults, err := searchHandler(keyword, proNames)