Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-06-24 13:55:05 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #5927 from steven-zou/return_labels_in_chart_api

Browse Source 
Refactor the chart service implementation to provide more extending flexibilities
This commit is contained in:
Wenkai Yin 2018-09-21 15:59:53 +08:00 committed by GitHub
commit 978f8721e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 1703 additions and 1605 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/chartserver/base_handler.go
View File

@ -1,17 +0,0 @@
package chartserver
import (
"net/http"
)
// BaseHandler defines the handlers related with the chart server itself.
type BaseHandler struct {
// Proxy used to to transfer the traffic of requests
// It's mainly used to talk to the backend chart server
trafficProxy *ProxyEngine
}
// GetHealthStatus will return the health status of the backend chart repository server
func (bh *BaseHandler) GetHealthStatus(w http.ResponseWriter, req *http.Request) {
bh.trafficProxy.ServeHTTP(w, req)
}

374
src/chartserver/base_test.go
View File

File diff suppressed because one or more lines are too long

15
src/chartserver/chart_operator.go
View File

@ -11,6 +11,7 @@ import (
"github.com/Masterminds/semver"
"github.com/goharbor/harbor/src/common/models"
hlog "github.com/goharbor/harbor/src/common/utils/log"
"k8s.io/helm/pkg/chartutil"
helm_repo "k8s.io/helm/pkg/repo"
@ -21,6 +22,15 @@ const (
valuesFileName = "values.yaml"
)
// ChartVersion extends the helm ChartVersion with additional labels
type ChartVersion struct {
helm_repo.ChartVersion
Labels []*models.Label `json:"labels"`
}
// ChartVersions is an array of extended ChartVersion
type ChartVersions []*ChartVersion
// ChartVersionDetails keeps the detailed data info of the chart version
type ChartVersionDetails struct {
Metadata *helm_repo.ChartVersion `json:"metadata"`
@ -28,6 +38,7 @@ type ChartVersionDetails struct {
Values map[string]interface{} `json:"values"`
Files map[string]string `json:"files"`
Security *SecurityReport `json:"security"`
Labels []*models.Label `json:"labels"`
}
// SecurityReport keeps the info related with security
@ -154,12 +165,12 @@ func (cho *ChartOperator) GetChartList(content []byte) ([]*ChartInfo, error) {
}
// GetChartVersions returns the chart versions
func (cho *ChartOperator) GetChartVersions(content []byte) (helm_repo.ChartVersions, error) {
func (cho *ChartOperator) GetChartVersions(content []byte) (ChartVersions, error) {
if content == nil || len(content) == 0 {
return nil, errors.New("zero content")
}
chartVersions := make(helm_repo.ChartVersions, 0)
chartVersions := make(ChartVersions, 0)
if err := json.Unmarshal(content, &chartVersions); err != nil {
return nil, err
}

6
src/chartserver/chart_operator_test.go
View File

@ -2,11 +2,13 @@ package chartserver
import (
"testing"
htesting "github.com/goharbor/harbor/src/testing"
)
func TestGetChartDetails(t *testing.T) {
chartOpr := ChartOperator{}
chartDetails, err := chartOpr.GetChartDetails(helmChartContent)
chartDetails, err := chartOpr.GetChartDetails(htesting.HelmChartContent)
if err != nil {
t.Fatal(err)
}
@ -26,7 +28,7 @@ func TestGetChartDetails(t *testing.T) {
func TestGetChartList(t *testing.T) {
chartOpr := ChartOperator{}
infos, err := chartOpr.GetChartList(chartListContent)
infos, err := chartOpr.GetChartList(htesting.ChartListContent)
if err != nil {
t.Fatal(err)
}

118
src/chartserver/controller.go
View File

@ -5,7 +5,6 @@ import (
"fmt"
"net/url"
"os"
"strings"
hlog "github.com/goharbor/harbor/src/common/utils/log"
)
@ -25,20 +24,21 @@ type Credential struct {
// A reverse proxy will be created and managed to proxy the related traffics between API and
// backend chart server
type Controller struct {
// Proxy used to to transfer the traffic of requests
// It's mainly used to talk to the backend chart server
trafficProxy *ProxyEngine
// Parse and process the chart version to provide required info data
chartOperator *ChartOperator
// HTTP client used to call the realted APIs of the backend chart repositories
apiClient *ChartClient
// The access endpoint of the backend chart repository server
backendServerAddr *url.URL
backendServerAddress *url.URL
// To cover the server info and status requests
baseHandler *BaseHandler
// To cover the chart repository requests
repositoryHandler *RepositoryHandler
// To cover all the manipulation requests
manipulationHandler *ManipulationHandler
// To cover the other utility requests
utilityHandler *UtilityHandler
// Cache the chart data
chartCache *ChartCache
}
// NewController is constructor of the chartserver.Controller
@ -53,15 +53,6 @@ func NewController(backendServer *url.URL) (*Controller, error) {
Password: os.Getenv(passwordKey),
}
// Use customized reverse proxy
proxy := NewProxyEngine(backendServer, cred)
// Create http client with customized timeouts
client := NewChartClient(cred)
// Initialize chart operator for use
operator := &ChartOperator{}
// Creat cache
cacheCfg, err := getCacheConfig()
if err != nil {
@ -75,79 +66,18 @@ func NewController(backendServer *url.URL) (*Controller, error) {
}
return &Controller{
backendServerAddr: backendServer,
baseHandler: &BaseHandler{proxy},
repositoryHandler: &RepositoryHandler{
trafficProxy: proxy,
apiClient: client,
backendServerAddress: backendServer,
},
manipulationHandler: &ManipulationHandler{
trafficProxy: proxy,
chartOperator: operator,
apiClient: client,
backendServerAddress: backendServer,
chartCache: cache,
},
utilityHandler: &UtilityHandler{
apiClient: client,
backendServerAddress: backendServer,
chartOperator: operator,
},
backendServerAddress: backendServer,
// Use customized reverse proxy
trafficProxy: NewProxyEngine(backendServer, cred),
// Initialize chart operator for use
chartOperator: &ChartOperator{},
// Create http client with customized timeouts
apiClient: NewChartClient(cred),
chartCache: cache,
}, nil
}
// GetBaseHandler returns the reference of BaseHandler
func (c *Controller) GetBaseHandler() *BaseHandler {
return c.baseHandler
}
// GetRepositoryHandler returns the reference of RepositoryHandler
func (c *Controller) GetRepositoryHandler() *RepositoryHandler {
return c.repositoryHandler
}
// GetManipulationHandler returns the reference of ManipulationHandler
func (c *Controller) GetManipulationHandler() *ManipulationHandler {
return c.manipulationHandler
}
// GetUtilityHandler returns the reference of UtilityHandler
func (c *Controller) GetUtilityHandler() *UtilityHandler {
return c.utilityHandler
}
// What's the cache driver if it is set
func parseCacheDriver() (string, bool) {
driver, ok := os.LookupEnv(cacheDriverENVKey)
return strings.ToLower(driver), ok
}
// Get and parse the configuration for the chart cache
func getCacheConfig() (*ChartCacheConfig, error) {
driver, isSet := parseCacheDriver()
if !isSet {
return nil, nil
}
if driver != cacheDriverMem && driver != cacheDriverRedis {
return nil, fmt.Errorf("cache driver '%s' is not supported, only support 'memory' and 'redis'", driver)
}
if driver == cacheDriverMem {
return &ChartCacheConfig{
DriverType: driver,
}, nil
}
redisConfigV := os.Getenv(redisENVKey)
redisCfg, err := parseRedisConfig(redisConfigV)
if err != nil {
return nil, fmt.Errorf("failed to parse redis configurations from '%s' with error: %s", redisCfg, err)
}
return &ChartCacheConfig{
DriverType: driver,
Config: redisCfg,
}, nil
// APIPrefix returns the API prefix path of calling backend chart service.
func (c *Controller) APIPrefix(namespace string) string {
return fmt.Sprintf("%s/api/%s/charts", c.backendServerAddress.String(), namespace)
}

220
src/chartserver/controller_test.go
View File

@ -1,225 +1,21 @@
package chartserver
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings"
"testing"
"github.com/ghodss/yaml"
helm_repo "k8s.io/helm/pkg/repo"
)
// Prepare, start the mock servers
func TestStartServers(t *testing.T) {
if err := startMockServers(); err != nil {
t.Fatal(err)
}
}
// Test /health
func TestGetHealthOfBaseHandler(t *testing.T) {
content, err := httpClient.GetContent(fmt.Sprintf("%s/health", getTheAddrOfFrontServer()))
// Test controller
func TestController(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
status := make(map[string]interface{})
if err := json.Unmarshal(content, &status); err != nil {
t.Fatalf("Unmarshal error: %s, %s", err, content)
}
healthy, ok := status["health"].(bool)
if !ok || !healthy {
t.Fatalf("Expect healthy of server to be 'true' but got %v", status["health"])
prefix := c.APIPrefix("fake")
expected := fmt.Sprintf("%s/api/%s/charts", s.URL, "fake")
if prefix != expected {
t.Fatalf("expect '%s' but got '%s'", expected, prefix)
}
}
// Get /repo1/index.yaml
func TestGetIndexYamlByRepo(t *testing.T) {
indexFile, err := getIndexYaml("/repo1/index.yaml")
if err != nil {
t.Fatal(err)
}
if len(indexFile.Entries) != 3 {
t.Fatalf("Expect index file with 3 entries, but got %d", len(indexFile.Entries))
}
}
// Test get /index.yaml
func TestGetUnifiedYamlFile(t *testing.T) {
indexFile, err := getIndexYaml("/index.yaml")
if err != nil {
t.Fatal(err)
}
if len(indexFile.Entries) != 5 {
t.Fatalf("Expect index file with 5 entries, but got %d", len(indexFile.Entries))
}
_, ok := indexFile.Entries["repo1/harbor"]
if !ok {
t.Fatal("Expect chart entry 'repo1/harbor' but got nothing")
}
_, ok = indexFile.Entries["repo2/harbor"]
if !ok {
t.Fatal("Expect chart entry 'repo2/harbor' but got nothing")
}
}
// Test download /:repo/charts/chart.tar
// Use this case to test the proxy function
func TestDownloadChart(t *testing.T) {
content, err := httpClient.GetContent(fmt.Sprintf("%s/repo1/charts/harbor-0.2.0.tgz", getTheAddrOfFrontServer()))
if err != nil {
t.Fatal(err)
}
gotSize := len(content)
expectSize := len(helmChartContent)
if gotSize != expectSize {
t.Fatalf("Expect %d bytes data but got %d bytes", expectSize, gotSize)
}
}
// Test get /api/:repo/charts
func TestRetrieveChartList(t *testing.T) {
content, err := httpClient.GetContent(fmt.Sprintf("%s/api/repo1/charts", getTheAddrOfFrontServer()))
if err != nil {
t.Fatal(err)
}
chartList := make([]*ChartInfo, 0)
err = json.Unmarshal(content, &chartList)
if err != nil {
t.Fatalf("Unmarshal error: %s", err)
}
if len(chartList) != 2 {
t.Fatalf("Expect to get 2 charts in the list but got %d", len(chartList))
}
foundItem := false
for _, chartInfo := range chartList {
if chartInfo.Name == "hello-helm" && chartInfo.TotalVersions == 2 {
foundItem = true
break
}
}
if !foundItem {
t.Fatalf("Expect chart 'hello-helm' with 2 versions but got nothing")
}
}
// Test get /api/:repo/charts/:chart_name/:version
func TestGetChartVersion(t *testing.T) {
content, err := httpClient.GetContent(fmt.Sprintf("%s/api/repo1/charts/harbor/0.2.0", getTheAddrOfFrontServer()))
if err != nil {
t.Fatal(err)
}
chartVersion := &ChartVersionDetails{}
if err = json.Unmarshal(content, chartVersion); err != nil {
t.Fatalf("Unmarshal error: %s", err)
}
if chartVersion.Metadata.Name != "harbor" {
t.Fatalf("Expect harbor chart version but got %s", chartVersion.Metadata.Name)
}
if chartVersion.Metadata.Version != "0.2.0" {
t.Fatalf("Expect version '0.2.0' but got version %s", chartVersion.Metadata.Version)
}
if len(chartVersion.Dependencies) != 1 {
t.Fatalf("Expect 1 dependency but got %d ones", len(chartVersion.Dependencies))
}
if len(chartVersion.Values) != 99 {
t.Fatalf("Expect 99 k-v values but got %d", len(chartVersion.Values))
}
}
// Test get /api/:repo/charts/:chart_name/:version with none-existing version
func TestGetChartVersionWithError(t *testing.T) {
_, err := httpClient.GetContent(fmt.Sprintf("%s/api/repo1/charts/harbor/1.0.0", getTheAddrOfFrontServer()))
if err == nil {
t.Fatal("Expect an error but got nil")
}
}
// Get /api/repo1/charts/harbor
// 401 will be rewritten to 500 with specified error
func TestResponseRewrite(t *testing.T) {
response, err := http.Get(fmt.Sprintf("%s/api/repo1/charts/harbor", getTheAddrOfFrontServer()))
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusInternalServerError {
t.Fatalf("Expect status code 500 but got %d", response.StatusCode)
}
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
t.Fatalf("Read bytes from http response failed with error: %s", err)
}
defer response.Body.Close()
errObj := make(map[string]interface{})
if err = json.Unmarshal(bytes, &errObj); err != nil {
t.Fatalf("Unmarshal error: %s", err)
}
if msg, ok := errObj["error"]; !ok {
t.Fatal("Expect an error message from server but got nothing")
} else {
if !strings.Contains(msg.(string), "operation request from unauthorized source is rejected") {
t.Fatal("Missing the required error message")
}
}
}
// Test the chart searching
func TestChartSearching(t *testing.T) {
namespaces := []string{"repo1", "repo2"}
q := "harbor"
results, err := mockController.GetRepositoryHandler().SearchChart(q, namespaces)
if err != nil {
t.Fatalf("expect nil error but got '%s'", err)
}
if len(results) != 2 {
t.Fatalf("expect 2 results but got %d", len(results))
}
}
// Clear environments
func TestStopServers(t *testing.T) {
stopMockServers()
}
// Utility method for getting index yaml file
func getIndexYaml(path string) (*helm_repo.IndexFile, error) {
content, err := httpClient.GetContent(fmt.Sprintf("%s%s", getTheAddrOfFrontServer(), path))
if err != nil {
return nil, err
}
indexFile := &helm_repo.IndexFile{}
if err := yaml.Unmarshal(content, indexFile); err != nil {
return nil, fmt.Errorf("Unmarshal error: %s", err)
}
if indexFile == nil {
return nil, fmt.Errorf("Got nil index yaml file")
}
return indexFile, nil
}

89
src/chartserver/handler_interface.go Normal file
View File

@ -0,0 +1,89 @@
package chartserver
import (
"net/http"
"k8s.io/helm/cmd/helm/search"
helm_repo "k8s.io/helm/pkg/repo"
)
// ServiceHandler defines the related methods to handle kinds of chart service requests.
type ServiceHandler interface {
// ListCharts lists all the charts under the specified namespace.
//
// namespace string: the chart namespace.
//
// If succeed, a chart info list with nil error will be returned;
// otherwise, a non-nil error will be got.
ListCharts(namespace string) ([]*ChartInfo, error)
// Get all the chart versions of the specified chart under the namespace.
//
// namespace string: the chart namespace.
// chartName string: the name of the chart, e.g: "harbor"
//
// If succeed, a chart version list with nil error will be returned;
// otherwise, a non-nil error will be got.
GetChart(namespace, chartName string) (helm_repo.ChartVersions, error)
// Get the detailed info of the specified chart version under the namespace.
// The detailed info includes chart summary, dependencies, values and signature status etc.
//
// namespace string: the chart namespace.
// chartName string: the name of the chart, e.g: "harbor"
// version string: the SemVer version of the chart, e.g: "0.2.0"
//
// If succeed, chart version details with nil error will be returned;
// otherwise, a non-nil error will be got.
GetChartVersionDetails(namespace, chartName, version string) (*ChartVersionDetails, error)
// SearchChart search charts in the specified namespaces with the keyword q.
// RegExp mode is enabled as default.
// For each chart, only the latest version will shown in the result list if matched to avoid duplicated entries.
// Keep consistent with `helm search` command.
//
// q string : the searching keyword
// namespaces []string : the search namespace scope
//
// If succeed, a search result list with nil error will be returned;
// otherwise, a non-nil error will be got.
SearchChart(q string, namespaces []string) ([]*search.Result, error)
// GetIndexFile will read the index.yaml under all namespaces and merge them as a single one
// Please be aware that, to support this function, the backend chart repository server should
// enable multi-tenancies
//
// namespaces []string : all the namespaces with accessing permissions
//
// If succeed, a unified merged index file with nil error will be returned;
// otherwise, a non-nil error will be got.
GetIndexFile(namespaces []string) (*helm_repo.IndexFile, error)
// Get the chart summary of the specified chart version.
//
// namespace string: the chart namespace.
// chartName string: the name of the chart, e.g: "harbor"
// version string: the SemVer version of the chart, e.g: "0.2.0"
//
// If succeed, chart version summary with nil error will be returned;
// otherwise, a non-nil error will be got.
GetChartVersion(namespace, name, version string) (*helm_repo.ChartVersion, error)
// DeleteChart deletes all the chart versions of the specified chart under the namespace.
//
// namespace string: the chart namespace.
// chartName string: the name of the chart, e.g: "harbor"
//
// If succeed, a nil error will be returned;
// otherwise, a non-nil error will be got.
DeleteChart(namespace, chartName string) error
}
// ProxyTrafficHandler defines the handler methods to handle the proxy traffic.
type ProxyTrafficHandler interface {
// Proxy the traffic to the backended server
//
// Req *http.Request : The incoming http request
// w http.ResponseWriter : The response writer reference
ProxyTraffic(w http.ResponseWriter, req *http.Request)
}

93
src/chartserver/handler_manipulation.go Normal file
View File

@ -0,0 +1,93 @@
package chartserver
import (
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/ghodss/yaml"
helm_repo "k8s.io/helm/pkg/repo"
)
// ListCharts gets the chart list under the namespace
// See @ServiceHandler.ListCharts
func (c *Controller) ListCharts(namespace string) ([]*ChartInfo, error) {
if len(strings.TrimSpace(namespace)) == 0 {
return nil, errors.New("empty namespace when getting chart list")
}
content, err := c.apiClient.GetContent(c.APIPrefix(namespace))
if err != nil {
return nil, err
}
return c.chartOperator.GetChartList(content)
}
// GetChart returns all the chart versions under the specified chart
// See @ServiceHandler.GetChart
func (c *Controller) GetChart(namespace, chartName string) (ChartVersions, error) {
if len(namespace) == 0 {
return nil, errors.New("empty name when getting chart versions")
}
if len(chartName) == 0 {
return nil, errors.New("no chart name specified")
}
url := fmt.Sprintf("%s/%s", c.APIPrefix(namespace), chartName)
data, err := c.apiClient.GetContent(url)
if err != nil {
return nil, err
}
versions := make(ChartVersions, 0)
if err := json.Unmarshal(data, &versions); err != nil {
return nil, err
}
return versions, nil
}
// DeleteChartVersion will delete the specified version of the chart
// See @ServiceHandler.DeleteChartVersion
func (c *Controller) DeleteChartVersion(namespace, chartName, version string) error {
if len(namespace) == 0 {
return errors.New("empty namespace when deleting chart version")
}
if len(chartName) == 0 || len(version) == 0 {
return errors.New("invalid chart for deleting")
}
url := fmt.Sprintf("%s/%s/%s", c.APIPrefix(namespace), chartName, version)
return c.apiClient.DeleteContent(url)
}
// GetChartVersion returns the summary of the specified chart version.
// See @ServiceHandler.GetChartVersion
func (c *Controller) GetChartVersion(namespace, name, version string) (*helm_repo.ChartVersion, error) {
if len(namespace) == 0 {
return nil, errors.New("empty namespace when getting summary of chart version")
}
if len(name) == 0 || len(version) == 0 {
return nil, errors.New("invalid chart when getting summary")
}
url := fmt.Sprintf("%s/%s/%s", c.APIPrefix(namespace), name, version)
content, err := c.apiClient.GetContent(url)
if err != nil {
return nil, err
}
chartVersion := &helm_repo.ChartVersion{}
if err := yaml.Unmarshal(content, chartVersion); err != nil {
return nil, err
}
return chartVersion, nil
}

88
src/chartserver/handler_manipulation_test.go Normal file
View File

@ -0,0 +1,88 @@
package chartserver
import (
"testing"
)
// Test get /api/:repo/charts/harbor
func TestGetChart(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
versions, err := c.GetChart("repo1", "harbor")
if err != nil {
t.Fatal(err)
}
if len(versions) != 2 {
t.Fatalf("expect 2 chart versions of harbor but got %d", len(versions))
}
}
// Test delete /api/:repo/charts/harbor/0.2.0
func TestDeleteChartVersion(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
if err := c.DeleteChartVersion("repo1", "harbor", "0.2.0"); err != nil {
t.Fatal(err)
}
}
// Test get /api/:repo/charts
func TestRetrieveChartList(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
chartList, err := c.ListCharts("repo1")
if err != nil {
t.Fatal(err)
}
if len(chartList) != 2 {
t.Fatalf("Expect to get 2 charts in the list but got %d", len(chartList))
}
foundItem := false
for _, chartInfo := range chartList {
if chartInfo.Name == "hello-helm" && chartInfo.TotalVersions == 2 {
foundItem = true
break
}
}
if !foundItem {
t.Fatalf("Expect chart 'hello-helm' with 2 versions but got nothing")
}
}
// Test the GetChartVersion in utility handler
func TestGetChartVersionSummary(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
chartV, err := c.GetChartVersion("repo1", "harbor", "0.2.0")
if err != nil {
t.Fatal(err)
}
if chartV.GetName() != "harbor" {
t.Fatalf("expect chart name 'harbor' but got '%s'", chartV.GetName())
}
if chartV.GetVersion() != "0.2.0" {
t.Fatalf("expect chart version '0.2.0' but got '%s'", chartV.GetVersion())
}
}

12
src/chartserver/handler_proxy_traffic.go Normal file
View File

@ -0,0 +1,12 @@
package chartserver
import (
"net/http"
)
// ProxyTraffic implements the interface method.
func (c *Controller) ProxyTraffic(w http.ResponseWriter, req *http.Request) {
if c.trafficProxy != nil {
c.trafficProxy.ServeHTTP(w, req)
}
}

137
src/chartserver/handler_proxy_traffic_test.go Normal file
View File

@ -0,0 +1,137 @@
package chartserver
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/ghodss/yaml"
htesting "github.com/goharbor/harbor/src/testing"
helm_repo "k8s.io/helm/pkg/repo"
)
// The frontend server
var frontServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mockController.ProxyTraffic(w, r)
}))
var mockServer *httptest.Server
var mockController *Controller
// Prepare case
func TestStartMockServers(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
mockController = c
mockServer = s
frontServer.Start()
}
// Test /health
func TestGetHealthOfBaseHandler(t *testing.T) {
content, err := httpClient.GetContent(fmt.Sprintf("%s/api/chartrepo/health", frontServer.URL))
if err != nil {
t.Fatal(err)
}
status := make(map[string]interface{})
if err := json.Unmarshal(content, &status); err != nil {
t.Fatalf("Unmarshal error: %s, %s", err, content)
}
healthy, ok := status["health"].(bool)
if !ok || !healthy {
t.Fatalf("Expect healthy of server to be 'true' but got %v", status["health"])
}
}
// Get /repo1/index.yaml
func TestGetIndexYamlByRepo(t *testing.T) {
indexFile, err := getIndexYaml("/chartrepo/repo1/index.yaml")
if err != nil {
t.Fatal(err)
}
if len(indexFile.Entries) != 3 {
t.Fatalf("Expect index file with 3 entries, but got %d", len(indexFile.Entries))
}
}
// Test download /:repo/charts/chart.tar
// Use this case to test the proxy function
func TestDownloadChart(t *testing.T) {
content, err := httpClient.GetContent(fmt.Sprintf("%s/chartrepo/repo1/charts/harbor-0.2.0.tgz", frontServer.URL))
if err != nil {
t.Fatal(err)
}
gotSize := len(content)
expectSize := len(htesting.HelmChartContent)
if gotSize != expectSize {
t.Fatalf("Expect %d bytes data but got %d bytes", expectSize, gotSize)
}
}
// Get /api/repo1/charts/harbor
// 401 will be rewritten to 500 with specified error
func TestResponseRewrite(t *testing.T) {
response, err := http.Get(fmt.Sprintf("%s/chartrepo/repo3/charts/harbor-0.8.1.tgz", frontServer.URL))
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusInternalServerError {
t.Fatalf("Expect status code 500 but got %d", response.StatusCode)
}
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
t.Fatalf("Read bytes from http response failed with error: %s", err)
}
defer response.Body.Close()
errObj := make(map[string]interface{})
if err = json.Unmarshal(bytes, &errObj); err != nil {
t.Fatalf("Unmarshal error: %s", err)
}
if msg, ok := errObj["error"]; !ok {
t.Fatal("Expect an error message from server but got nothing")
} else {
if !strings.Contains(msg.(string), "operation request from unauthorized source is rejected") {
t.Fatal("Missing the required error message")
}
}
}
// Clear env
func TestStopMockServers(t *testing.T) {
frontServer.Close()
mockServer.Close()
}
// Utility method for getting index yaml file
func getIndexYaml(path string) (*helm_repo.IndexFile, error) {
content, err := httpClient.GetContent(fmt.Sprintf("%s%s", frontServer.URL, path))
if err != nil {
return nil, err
}
indexFile := &helm_repo.IndexFile{}
if err := yaml.Unmarshal(content, indexFile); err != nil {
return nil, fmt.Errorf("Unmarshal error: %s", err)
}
if indexFile == nil {
return nil, fmt.Errorf("Got nil index yaml file")
}
return indexFile, nil
}

133
src/chartserver/repo_handler.go → src/chartserver/handler_repo.go
View File

@ -1,17 +1,12 @@
package chartserver
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"path"
"sync"
"time"
"github.com/ghodss/yaml"
"github.com/goharbor/harbor/src/core/filter"
"k8s.io/helm/cmd/helm/search"
helm_repo "k8s.io/helm/pkg/repo"
hlog "github.com/goharbor/harbor/src/common/utils/log"
@ -24,124 +19,33 @@ const (
searchMaxScore = 25
)
// RepositoryHandler defines all the handlers to handle the requests related with chart repository
// e.g: index.yaml and downloading chart objects
type RepositoryHandler struct {
// Proxy used to to transfer the traffic of requests
// It's mainly used to talk to the backend chart server
trafficProxy *ProxyEngine
// HTTP client used to call the realted APIs of the backend chart repositories
apiClient *ChartClient
// Point to the url of the backend server
backendServerAddress *url.URL
}
// Result returned by worker
type processedResult struct {
namespace string
indexFileOfRepo *helm_repo.IndexFile
}
// GetIndexFileWithNS will read the index.yaml data under the specified namespace
func (rh *RepositoryHandler) GetIndexFileWithNS(w http.ResponseWriter, req *http.Request) {
rh.trafficProxy.ServeHTTP(w, req)
}
// GetIndexFile will read the index.yaml under all namespaces and merge them as a single one
// Please be aware that, to support this function, the backend chart repository server should
// enable multi-tenancies
func (rh *RepositoryHandler) GetIndexFile(w http.ResponseWriter, req *http.Request) {
// Get project manager references
projectMgr, err := filter.GetProjectManager(req)
if err != nil {
WriteInternalError(w, err)
return
//
// See @ServiceHandler.GetIndexFile
func (c *Controller) GetIndexFile(namespaces []string) (*helm_repo.IndexFile, error) {
if namespaces == nil || len(namespaces) == 0 {
return emptyIndexFile(), nil
}
// Get all the projects
results, err := projectMgr.List(nil)
if err != nil {
WriteInternalError(w, err)
return
}
// If no projects existing, return empty index.yaml content immediately
if results.Total == 0 {
w.Write(emptyIndexFile())
return
}
namespaces := []string{}
for _, p := range results.Projects {
namespaces = append(namespaces, p.Name)
}
mergedIndexFile, err := rh.getIndexYaml(namespaces)
if err != nil {
WriteInternalError(w, err)
return
}
bytes, err := yaml.Marshal(mergedIndexFile)
if err != nil {
WriteInternalError(w, err)
return
}
w.Write(bytes)
}
// DownloadChartObject will download the stored chart object to the client
// e.g: helm install
func (rh *RepositoryHandler) DownloadChartObject(w http.ResponseWriter, req *http.Request) {
rh.trafficProxy.ServeHTTP(w, req)
}
// SearchChart search charts in the specified namespaces with the keyword q.
// RegExp mode is enabled as default.
// For each chart, only the latest version will shown in the result list if matched to avoid duplicated entries.
// Keep consistent with `helm search` command.
func (rh *RepositoryHandler) SearchChart(q string, namespaces []string) ([]*search.Result, error) {
if len(q) == 0 || len(namespaces) == 0 {
// Return empty list
return []*search.Result{}, nil
}
// Get the merged index yaml file of the namespaces
ind, err := rh.getIndexYaml(namespaces)
if err != nil {
return nil, err
}
// Build the search index
index := search.NewIndex()
// As the repo name is already merged into the index yaml, we use empty repo name.
// Set 'All' to false to return only one version for each chart.
index.AddRepo("", ind, false)
// Search
// RegExp is enabled
results, err := index.Search(q, searchMaxScore, true)
if err != nil {
return nil, err
}
// Sort results.
search.SortScore(results)
return results, nil
return c.getIndexYaml(namespaces)
}
// getIndexYaml will get the index yaml files for all the namespaces and merge them
// as one unified index yaml file.
func (rh *RepositoryHandler) getIndexYaml(namespaces []string) (*helm_repo.IndexFile, error) {
func (c *Controller) getIndexYaml(namespaces []string) (*helm_repo.IndexFile, error) {
// The final merged index file
mergedIndexFile := &helm_repo.IndexFile{
APIVersion: "v1",
Entries: make(map[string]helm_repo.ChartVersions),
Generated: time.Now(),
Generated: time.Now().Round(time.Second),
PublicKeys: []string{},
}
@ -175,7 +79,7 @@ func (rh *RepositoryHandler) getIndexYaml(namespaces []string) (*helm_repo.Index
}()
for res := range resultChan {
rh.mergeIndexFile(res.namespace, mergedIndexFile, res.indexFileOfRepo)
c.mergeIndexFile(res.namespace, mergedIndexFile, res.indexFileOfRepo)
}
}()
@ -203,7 +107,7 @@ LOOP:
workerPool <- struct{}{}
}()
indexFile, err := rh.getIndexYamlWithNS(ns)
indexFile, err := c.getIndexYamlWithNS(ns)
if err != nil {
if len(errorChan) == 0 {
// Only need one error as failure signal
@ -254,13 +158,13 @@ LOOP:
}
// Get the index yaml file under the specified namespace from the backend server
func (rh *RepositoryHandler) getIndexYamlWithNS(namespace string) (*helm_repo.IndexFile, error) {
func (c *Controller) getIndexYamlWithNS(namespace string) (*helm_repo.IndexFile, error) {
// Join url path
url := path.Join(namespace, "index.yaml")
url = fmt.Sprintf("%s/%s", rh.backendServerAddress.String(), url)
url = fmt.Sprintf("%s/%s", c.backendServerAddress.String(), url)
hlog.Debugf("Getting index.yaml from '%s'", url)
content, err := rh.apiClient.GetContent(url)
content, err := c.apiClient.GetContent(url)
if err != nil {
return nil, err
}
@ -276,7 +180,7 @@ func (rh *RepositoryHandler) getIndexYamlWithNS(namespace string) (*helm_repo.In
// Merge the content of mergingIndexFile to the baseIndex
// The chart url should be without --chart-url prefix
func (rh *RepositoryHandler) mergeIndexFile(namespace string,
func (c *Controller) mergeIndexFile(namespace string,
baseIndex *helm_repo.IndexFile,
mergingIndexFile *helm_repo.IndexFile) {
// Append entries
@ -304,12 +208,9 @@ func (rh *RepositoryHandler) mergeIndexFile(namespace string,
}
// Generate empty index file
func emptyIndexFile() []byte {
func emptyIndexFile() *helm_repo.IndexFile {
emptyIndexFile := &helm_repo.IndexFile{}
emptyIndexFile.Generated = time.Now()
emptyIndexFile.Generated = time.Now().Round(time.Second)
// Ignore the error
rawData, _ := json.Marshal(emptyIndexFile)
return rawData
return emptyIndexFile
}

32
src/chartserver/handler_repo_test.go Normal file
View File

@ -0,0 +1,32 @@
package chartserver
import "testing"
// Test get /index.yaml
func TestGetIndexFile(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
namespaces := []string{"repo1", "repo2"}
indexFile, err := c.GetIndexFile(namespaces)
if err != nil {
t.Fatal(err)
}
if len(indexFile.Entries) != 5 {
t.Fatalf("Expect index file with 5 entries, but got %d", len(indexFile.Entries))
}
_, ok := indexFile.Entries["repo1/harbor"]
if !ok {
t.Fatal("Expect chart entry 'repo1/harbor' but got nothing")
}
_, ok = indexFile.Entries["repo2/harbor"]
if !ok {
t.Fatal("Expect chart entry 'repo2/harbor' but got nothing")
}
}

208
src/chartserver/handler_utility.go Normal file
View File

@ -0,0 +1,208 @@
package chartserver
import (
"errors"
"fmt"
"path"
"strings"
"sync"
hlog "github.com/goharbor/harbor/src/common/utils/log"
"k8s.io/helm/cmd/helm/search"
)
const (
maxDeletionThreads = 10
)
// DeleteChart deletes all the chart versions of the specified chart under the namespace.
// See @ServiceHandler.DeleteChart
func (c *Controller) DeleteChart(namespace, chartName string) error {
if len(strings.TrimSpace(namespace)) == 0 {
return errors.New("empty namespace when deleting chart")
}
if len(strings.TrimSpace(chartName)) == 0 {
return errors.New("empty chart name when deleting chart")
}
url := fmt.Sprintf("%s/%s", c.APIPrefix(namespace), chartName)
content, err := c.apiClient.GetContent(url)
if err != nil {
return err
}
allVersions, err := c.chartOperator.GetChartVersions(content)
if err != nil {
return err
}
// Let's delete the versions in parallel
// The number of goroutine is controlled by the const maxDeletionThreads
qSize := len(allVersions)
if qSize > maxDeletionThreads {
qSize = maxDeletionThreads
}
tokenQueue := make(chan struct{}, qSize)
errChan := make(chan error, 1)
waitGroup := new(sync.WaitGroup)
waitGroup.Add(len(allVersions))
// Append initial tokens
for i := 0; i < qSize; i++ {
tokenQueue <- struct{}{}
}
// Collect errors
errs := make([]error, 0)
errWrapper := make(chan error, 1)
go func() {
defer func() {
// pass to the out func
if len(errs) > 0 {
errWrapper <- fmt.Errorf("%v", errs)
}
close(errWrapper)
}()
for deletionErr := range errChan {
errs = append(errs, deletionErr)
}
}()
// Schedule deletion tasks
for _, deletingVersion := range allVersions {
// Apply for token first
// If no available token, pending here
<-tokenQueue
// Got one token
go func(deletingVersion *ChartVersion) {
defer func() {
// return the token back
tokenQueue <- struct{}{}
// done
waitGroup.Done()
}()
if err := c.DeleteChartVersion(namespace, chartName, deletingVersion.GetVersion()); err != nil {
errChan <- err
}
}(deletingVersion)
}
// Wait all goroutines are done
waitGroup.Wait()
// Safe to quit error collection goroutine
close(errChan)
err = <-errWrapper
return err
}
// GetChartVersionDetails get the specified version for one chart
// This handler should return the details of the chart version,
// maybe including metadata,dependencies and values etc.
// See @ServiceHandler.GetChartVersionDetails
func (c *Controller) GetChartVersionDetails(namespace, chartName, version string) (*ChartVersionDetails, error) {
chartV, err := c.GetChartVersion(namespace, chartName, version)
if err != nil {
return nil, err
}
// Query cache
chartDetails := c.chartCache.GetChart(chartV.Digest)
if chartDetails == nil {
// NOT hit!!
content, err := c.getChartVersionContent(namespace, chartV.URLs[0])
if err != nil {
return nil, err
}
// Process bytes and get more details of chart version
chartDetails, err = c.chartOperator.GetChartDetails(content)
if err != nil {
return nil, err
}
chartDetails.Metadata = chartV
// Put it into the cache for next access
c.chartCache.PutChart(chartDetails)
} else {
// Just logged
hlog.Debugf("Get detailed data from cache for chart: %s:%s (%s)",
chartDetails.Metadata.Name,
chartDetails.Metadata.Version,
chartDetails.Metadata.Digest)
}
// The change of prov file will not cause any influence to the digest of chart,
// and then the digital signature status should be not cached
//
// Generate the security report
// prov file share same endpoint with the chart version
// Just add .prov suffix to the chart version to form the path of prov file
// Anyway, there will be a report about the digital signature status
chartDetails.Security = &SecurityReport{
Signature: &DigitalSignature{
Signed: false,
},
}
// Try to get the prov file to confirm if it is exitsing
provFilePath := fmt.Sprintf("%s.prov", chartV.URLs[0])
provBytes, err := c.getChartVersionContent(namespace, provFilePath)
if err == nil && len(provBytes) > 0 {
chartDetails.Security.Signature.Signed = true
chartDetails.Security.Signature.Provenance = provFilePath
} else {
// Just log it
hlog.Errorf("Failed to get prov file for chart %s with error: %s, got %d bytes", chartV.Name, err.Error(), len(provBytes))
}
return chartDetails, nil
}
// SearchChart search charts in the specified namespaces with the keyword q.
// RegExp mode is enabled as default.
// For each chart, only the latest version will shown in the result list if matched to avoid duplicated entries.
// Keep consistent with `helm search` command.
func (c *Controller) SearchChart(q string, namespaces []string) ([]*search.Result, error) {
if len(q) == 0 || len(namespaces) == 0 {
// Return empty list
return []*search.Result{}, nil
}
// Get the merged index yaml file of the namespaces
ind, err := c.getIndexYaml(namespaces)
if err != nil {
return nil, err
}
// Build the search index
index := search.NewIndex()
// As the repo name is already merged into the index yaml, we use empty repo name.
// Set 'All' to false to return only one version for each chart.
index.AddRepo("", ind, false)
// Search
// RegExp is enabled
results, err := index.Search(q, searchMaxScore, true)
if err != nil {
return nil, err
}
// Sort results.
search.SortScore(results)
return results, nil
}
// Get the content bytes of the chart version
func (c *Controller) getChartVersionContent(namespace string, subPath string) ([]byte, error) {
url := path.Join(namespace, subPath)
url = fmt.Sprintf("%s/%s", c.backendServerAddress.String(), url)
return c.apiClient.GetContent(url)
}

83
src/chartserver/handler_utility_test.go Normal file
View File

@ -0,0 +1,83 @@
package chartserver
import (
"testing"
)
// Test the function DeleteChart
func TestDeleteChart(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
if err := c.DeleteChart("repo1", "harbor"); err != nil {
t.Fatal(err)
}
}
// Test get /api/:repo/charts/:chart_name/:version
func TestGetChartVersion(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
chartVersion, err := c.GetChartVersionDetails("repo1", "harbor", "0.2.0")
if err != nil {
t.Fatal(err)
}
if chartVersion.Metadata.Name != "harbor" {
t.Fatalf("Expect harbor chart version but got %s", chartVersion.Metadata.Name)
}
if chartVersion.Metadata.Version != "0.2.0" {
t.Fatalf("Expect version '0.2.0' but got version %s", chartVersion.Metadata.Version)
}
if len(chartVersion.Dependencies) != 1 {
t.Fatalf("Expect 1 dependency but got %d ones", len(chartVersion.Dependencies))
}
if len(chartVersion.Values) != 99 {
t.Fatalf("Expect 99 k-v values but got %d", len(chartVersion.Values))
}
}
// Test get /api/:repo/charts/:chart_name/:version with none-existing version
func TestGetChartVersionWithError(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
_, err = c.GetChartVersionDetails("repo1", "harbor", "1.0.0")
if err == nil {
t.Fatal("Expect an error but got nil")
}
}
// Test the chart searching
func TestChartSearching(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
namespaces := []string{"repo1", "repo2"}
q := "harbor"
results, err := c.SearchChart(q, namespaces)
if err != nil {
t.Fatalf("expect nil error but got '%s'", err)
}
if len(results) != 2 {
t.Fatalf("expect 2 results but got %d", len(results))
}
}

199
src/chartserver/manipulation_handler.go
View File

@ -1,199 +0,0 @@
package chartserver
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"path"
"strings"
"github.com/ghodss/yaml"
hlog "github.com/goharbor/harbor/src/common/utils/log"
helm_repo "k8s.io/helm/pkg/repo"
)
const (
// NamespaceContextKey is context key for the namespace
NamespaceContextKey ContextKey = ":repo"
)
// ContextKey is defined for add value in the context of http request
type ContextKey string
// ManipulationHandler includes all the handler methods for the purpose of manipulating the
// chart repository
type ManipulationHandler struct {
// Proxy used to to transfer the traffic of requests
// It's mainly used to talk to the backend chart server
trafficProxy *ProxyEngine
// Parse and process the chart version to provide required info data
chartOperator *ChartOperator
// HTTP client used to call the realted APIs of the backend chart repositories
apiClient *ChartClient
// Point to the url of the backend server
backendServerAddress *url.URL
// Cache the chart data
chartCache *ChartCache
}
// ListCharts lists all the charts under the specified namespace
func (mh *ManipulationHandler) ListCharts(w http.ResponseWriter, req *http.Request) {
url := strings.TrimPrefix(req.URL.String(), "/")
url = fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), url)
content, err := mh.apiClient.GetContent(url)
if err != nil {
WriteInternalError(w, err)
return
}
chartList, err := mh.chartOperator.GetChartList(content)
if err != nil {
WriteInternalError(w, err)
return
}
jsonData, err := json.Marshal(chartList)
if err != nil {
WriteInternalError(w, err)
return
}
writeJSONData(w, jsonData)
}
// GetChart returns all the chart versions under the specified chart
func (mh *ManipulationHandler) GetChart(w http.ResponseWriter, req *http.Request) {
mh.trafficProxy.ServeHTTP(w, req)
}
// GetChartVersion get the specified version for one chart
// This handler should return the details of the chart version,
// maybe including metadata,dependencies and values etc.
func (mh *ManipulationHandler) GetChartVersion(w http.ResponseWriter, req *http.Request) {
chartV, err := mh.getChartVersion(req.URL.String())
if err != nil {
WriteInternalError(w, err)
return
}
// Get and check namespace
// even we get the data from cache
var namespace string
repoValue := req.Context().Value(NamespaceContextKey)
if repoValue != nil {
if ns, ok := repoValue.(string); ok {
namespace = ns
}
}
if len(strings.TrimSpace(namespace)) == 0 {
WriteInternalError(w, errors.New("failed to extract namespace from the request"))
return
}
// Query cache
chartDetails := mh.chartCache.GetChart(chartV.Digest)
if chartDetails == nil {
// NOT hit!!
content, err := mh.getChartVersionContent(namespace, chartV.URLs[0])
if err != nil {
WriteInternalError(w, err)
return
}
// Process bytes and get more details of chart version
chartDetails, err = mh.chartOperator.GetChartDetails(content)
if err != nil {
WriteInternalError(w, err)
return
}
chartDetails.Metadata = chartV
// Put it into the cache for next access
mh.chartCache.PutChart(chartDetails)
} else {
// Just logged
hlog.Debugf("Get detailed data from cache for chart: %s:%s (%s)",
chartDetails.Metadata.Name,
chartDetails.Metadata.Version,
chartDetails.Metadata.Digest)
}
// The change of prov file will not cause any influence to the digest of chart,
// and then the digital signature status should be not cached
//
// Generate the security report
// prov file share same endpoint with the chart version
// Just add .prov suffix to the chart version to form the path of prov file
// Anyway, there will be a report about the digital signature status
chartDetails.Security = &SecurityReport{
Signature: &DigitalSignature{
Signed: false,
},
}
// Try to get the prov file to confirm if it is exitsing
provFilePath := fmt.Sprintf("%s.prov", chartV.URLs[0])
provBytes, err := mh.getChartVersionContent(namespace, provFilePath)
if err == nil && len(provBytes) > 0 {
chartDetails.Security.Signature.Signed = true
chartDetails.Security.Signature.Provenance = provFilePath
} else {
// Just log it
hlog.Errorf("Failed to get prov file for chart %s with error: %s, got %d bytes", chartV.Name, err.Error(), len(provBytes))
}
bytes, err := json.Marshal(chartDetails)
if err != nil {
WriteInternalError(w, err)
return
}
writeJSONData(w, bytes)
}
// UploadChartVersion will save the new version of the chart to the backend storage
func (mh *ManipulationHandler) UploadChartVersion(w http.ResponseWriter, req *http.Request) {
mh.trafficProxy.ServeHTTP(w, req)
}
// UploadProvenanceFile will save the provenance file of the chart to the backend storage
func (mh *ManipulationHandler) UploadProvenanceFile(w http.ResponseWriter, req *http.Request) {
mh.trafficProxy.ServeHTTP(w, req)
}
// DeleteChartVersion will delete the specified version of the chart
func (mh *ManipulationHandler) DeleteChartVersion(w http.ResponseWriter, req *http.Request) {
mh.trafficProxy.ServeHTTP(w, req)
}
// Get the basic metadata of chart version
func (mh *ManipulationHandler) getChartVersion(subPath string) (*helm_repo.ChartVersion, error) {
url := fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), strings.TrimPrefix(subPath, "/"))
content, err := mh.apiClient.GetContent(url)
if err != nil {
return nil, err
}
chartVersion := &helm_repo.ChartVersion{}
if err := yaml.Unmarshal(content, chartVersion); err != nil {
return nil, err
}
return chartVersion, nil
}
// Get the content bytes of the chart version
func (mh *ManipulationHandler) getChartVersionContent(namespace string, subPath string) ([]byte, error) {
url := path.Join(namespace, subPath)
url = fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), url)
return mh.apiClient.GetContent(url)
}

37
src/chartserver/reverse_proxy.go
View File

@ -17,6 +17,11 @@ import (
const (
agentHarbor = "HARBOR"
contentLengthHeader = "Content-Length"
defaultRepo = "library"
rootUploadingEndpoint = "/api/chartrepo/charts"
rootIndexEndpoint = "/chartrepo/index.yaml"
chartRepoHealthEndpoint = "/api/chartrepo/health"
)
// ProxyEngine is used to proxy the related traffics
@ -56,6 +61,7 @@ func director(target *url.URL, cred *Credential, req *http.Request) {
// Overwrite the request URL to the target path
req.URL.Scheme = target.Scheme
req.URL.Host = target.Host
rewriteURLPath(req)
req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path)
if targetQuery == "" || req.URL.RawQuery == "" {
req.URL.RawQuery = targetQuery + req.URL.RawQuery
@ -125,3 +131,34 @@ func singleJoiningSlash(a, b string) string {
}
return a + b
}
// Rewrite the incoming URL with the right backend URL pattern
// Remove 'chartrepo' from the endpoints of manipulation API
// Remove 'chartrepo' from the endpoints of repository services
func rewriteURLPath(req *http.Request) {
incomingURLPath := req.URL.Path
// Health check endpoint
if incomingURLPath == chartRepoHealthEndpoint {
req.URL.Path = "/health"
return
}
// Root uploading endpoint
if incomingURLPath == rootUploadingEndpoint {
req.URL.Path = strings.Replace(incomingURLPath, "chartrepo", defaultRepo, 1)
return
}
// Repository endpoints
if strings.HasPrefix(incomingURLPath, "/chartrepo") {
req.URL.Path = strings.TrimPrefix(incomingURLPath, "/chartrepo")
return
}
// API endpoints
if strings.HasPrefix(incomingURLPath, "/api/chartrepo") {
req.URL.Path = strings.Replace(incomingURLPath, "/chartrepo", "", 1)
return
}
}

55
src/chartserver/reverse_proxy_test.go Normal file
View File

@ -0,0 +1,55 @@
package chartserver
import (
"net/http"
"testing"
)
// Test the URL rewrite function
func TestURLRewrite(t *testing.T) {
req, err := createRequest(http.MethodGet, "/api/chartrepo/health")
if err != nil {
t.Fatal(err)
}
rewriteURLPath(req)
if req.URL.Path != "/health" {
t.Fatalf("Expect url format %s but got %s", "/health", req.URL.Path)
}
req, err = createRequest(http.MethodGet, "/api/chartrepo/library/charts")
if err != nil {
t.Fatal(err)
}
rewriteURLPath(req)
if req.URL.Path != "/api/library/charts" {
t.Fatalf("Expect url format %s but got %s", "/api/library/charts", req.URL.Path)
}
req, err = createRequest(http.MethodPost, "/api/chartrepo/charts")
if err != nil {
t.Fatal(err)
}
rewriteURLPath(req)
if req.URL.Path != "/api/library/charts" {
t.Fatalf("Expect url format %s but got %s", "/api/library/charts", req.URL.Path)
}
req, err = createRequest(http.MethodGet, "/chartrepo/library/index.yaml")
if err != nil {
t.Fatal(err)
}
rewriteURLPath(req)
if req.URL.Path != "/library/index.yaml" {
t.Fatalf("Expect url format %s but got %s", "/library/index.yaml", req.URL.Path)
}
}
func createRequest(method string, url string) (*http.Request, error) {
req, err := http.NewRequest(method, url, nil)
if err != nil {
return nil, err
}
req.RequestURI = url
return req, nil
}

163
src/chartserver/utility_handler.go
View File

@ -1,163 +0,0 @@
package chartserver
import (
"errors"
"fmt"
"net/url"
"strings"
"sync"
"github.com/ghodss/yaml"
helm_repo "k8s.io/helm/pkg/repo"
)
const (
maxDeletionThreads = 10
)
// UtilityHandler provides utility methods
type UtilityHandler struct {
// Parse and process the chart version to provide required info data
chartOperator *ChartOperator
// HTTP client used to call the realted APIs of the backend chart repositories
apiClient *ChartClient
// Point to the url of the backend server
backendServerAddress *url.URL
}
// GetChartsByNs gets the chart list under the namespace
func (uh *UtilityHandler) GetChartsByNs(namespace string) ([]*ChartInfo, error) {
if len(strings.TrimSpace(namespace)) == 0 {
return nil, errors.New("empty namespace when getting chart list")
}
path := fmt.Sprintf("/api/%s/charts", namespace)
url := fmt.Sprintf("%s%s", uh.backendServerAddress.String(), path)
content, err := uh.apiClient.GetContent(url)
if err != nil {
return nil, err
}
return uh.chartOperator.GetChartList(content)
}
// DeleteChart deletes all the chart versions of the specified chart under the namespace.
func (uh *UtilityHandler) DeleteChart(namespace, chartName string) error {
if len(strings.TrimSpace(namespace)) == 0 {
return errors.New("empty namespace when deleting chart")
}
if len(strings.TrimSpace(chartName)) == 0 {
return errors.New("empty chart name when deleting chart")
}
path := fmt.Sprintf("/api/%s/charts/%s", namespace, chartName)
url := fmt.Sprintf("%s%s", uh.backendServerAddress.String(), path)
content, err := uh.apiClient.GetContent(url)
if err != nil {
return err
}
allVersions, err := uh.chartOperator.GetChartVersions(content)
if err != nil {
return err
}
// Let's delete the versions in parallel
// The number of goroutine is controlled by the const maxDeletionThreads
qSize := len(allVersions)
if qSize > maxDeletionThreads {
qSize = maxDeletionThreads
}
tokenQueue := make(chan struct{}, qSize)
errChan := make(chan error, 1)
waitGroup := new(sync.WaitGroup)
waitGroup.Add(len(allVersions))
// Append initial tokens
for i := 0; i < qSize; i++ {
tokenQueue <- struct{}{}
}
// Collect errors
errs := make([]error, 0)
errWrapper := make(chan error, 1)
go func() {
defer func() {
// pass to the out func
if len(errs) > 0 {
errWrapper <- fmt.Errorf("%v", errs)
}
close(errWrapper)
}()
for deletionErr := range errChan {
errs = append(errs, deletionErr)
}
}()
// Schedule deletion tasks
for _, deletingVersion := range allVersions {
// Apply for token first
// If no available token, pending here
<-tokenQueue
// Got one token
go func(deletingVersion *helm_repo.ChartVersion) {
defer func() {
// return the token back
tokenQueue <- struct{}{}
// done
waitGroup.Done()
}()
if err := uh.deleteChartVersion(namespace, chartName, deletingVersion.GetVersion()); err != nil {
errChan <- err
}
}(deletingVersion)
}
// Wait all goroutines are done
waitGroup.Wait()
// Safe to quit error collection goroutine
close(errChan)
err = <-errWrapper
return err
}
// GetChartVersion returns the summary of the specified chart version.
func (uh *UtilityHandler) GetChartVersion(namespace, name, version string) (*helm_repo.ChartVersion, error) {
if len(namespace) == 0 || len(name) == 0 || len(version) == 0 {
return nil, errors.New("bad arguments to get chart version summary")
}
path := fmt.Sprintf("/api/%s/charts/%s/%s", namespace, name, version)
url := fmt.Sprintf("%s%s", uh.backendServerAddress.String(), path)
content, err := uh.apiClient.GetContent(url)
if err != nil {
return nil, err
}
chartVersion := &helm_repo.ChartVersion{}
if err := yaml.Unmarshal(content, chartVersion); err != nil {
return nil, err
}
return chartVersion, nil
}
// deleteChartVersion deletes the specified chart version
func (uh *UtilityHandler) deleteChartVersion(namespace, chartName, version string) error {
path := fmt.Sprintf("/api/%s/charts/%s/%s", namespace, chartName, version)
url := fmt.Sprintf("%s%s", uh.backendServerAddress.String(), path)
return uh.apiClient.DeleteContent(url)
}

228
src/chartserver/utility_handler_test.go
View File

@ -1,228 +0,0 @@
package chartserver
import (
"net/http"
"net/http/httptest"
"net/url"
"testing"
)
// TestGetChartsByNs tests GetChartsByNs method in UtilityHandler
func TestGetChartsByNs(t *testing.T) {
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.RequestURI {
case "/api/repo1/charts":
if r.Method == http.MethodGet {
w.Write(chartListContent)
return
}
}
w.WriteHeader(http.StatusNotImplemented)
w.Write([]byte("not supported"))
}))
defer mockServer.Close()
serverURL, err := url.Parse(mockServer.URL)
if err != nil {
t.Fatal(err)
}
theController, err := NewController(serverURL)
if err != nil {
t.Fatal(err)
}
charts, err := theController.GetUtilityHandler().GetChartsByNs("repo1")
if err != nil {
t.Fatal(err)
}
if len(charts) != 2 {
t.Fatalf("expect 2 items but got %d", len(charts))
}
}
// Test the function DeleteChart
func TestDeleteChart(t *testing.T) {
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.RequestURI {
case "/api/repo1/charts/harbor":
if r.Method == http.MethodGet {
w.Write([]byte(chartVersionsOfHarbor))
return
}
case "/api/repo1/charts/harbor/0.2.0",
"/api/repo1/charts/harbor/0.2.1":
if r.Method == http.MethodDelete {
w.WriteHeader(http.StatusOK)
return
}
}
w.WriteHeader(http.StatusNotImplemented)
w.Write([]byte("not supported"))
}))
defer mockServer.Close()
serverURL, err := url.Parse(mockServer.URL)
if err != nil {
t.Fatal(err)
}
theController, err := NewController(serverURL)
if err != nil {
t.Fatal(err)
}
if err := theController.GetUtilityHandler().DeleteChart("repo1", "harbor"); err != nil {
t.Fatal(err)
}
}
// Test the GetChartVersion in utility handler
func TestGetChartVersionSummary(t *testing.T) {
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.RequestURI {
case "/api/repo1/charts/harbor/0.2.0":
if r.Method == http.MethodGet {
w.Write([]byte(chartVersionOfHarbor020))
return
}
}
w.WriteHeader(http.StatusNotImplemented)
w.Write([]byte("not supported"))
}))
defer mockServer.Close()
serverURL, err := url.Parse(mockServer.URL)
if err != nil {
t.Fatal(err)
}
theController, err := NewController(serverURL)
if err != nil {
t.Fatal(err)
}
chartV, err := theController.GetUtilityHandler().GetChartVersion("repo1", "harbor", "0.2.0")
if err != nil {
t.Fatal(err)
}
if chartV.GetName() != "harbor" {
t.Fatalf("expect chart name 'harbor' but got '%s'", chartV.GetName())
}
if chartV.GetVersion() != "0.2.0" {
t.Fatalf("expect chart version '0.2.0' but got '%s'", chartV.GetVersion())
}
}
var chartVersionOfHarbor020 = `
{
"name": "harbor",
"home": "https://github.com/vmware/harbor",
"sources": [
"https://github.com/vmware/harbor/tree/master/contrib/helm/harbor"
],
"version": "0.2.0",
"description": "An Enterprise-class Docker Registry by VMware",
"keywords": [
"vmware",
"docker",
"registry",
"harbor"
],
"maintainers": [
{
"name": "Jesse Hu",
"email": "huh@vmware.com"
},
{
"name": "paulczar",
"email": "username.taken@gmail.com"
}
],
"engine": "gotpl",
"icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png",
"appVersion": "1.5.0",
"urls": [
"charts/harbor-0.2.0.tgz"
],
"created": "2018-08-29T10:26:21.141611102Z",
"digest": "fc8aae8dade9f0dfca12e9f1085081c49843d30a063a3fa7eb42497e3ceb277c"
}
`
var chartVersionsOfHarbor = `
[
{
"name": "harbor",
"home": "https://github.com/vmware/harbor",
"sources": [
"https://github.com/vmware/harbor/tree/master/contrib/helm/harbor"
],
"version": "0.2.1",
"description": "An Enterprise-class Docker Registry by VMware",
"keywords": [
"vmware",
"docker",
"registry",
"harbor"
],
"maintainers": [
{
"name": "Jesse Hu",
"email": "huh@vmware.com"
},
{
"name": "paulczar",
"email": "username.taken@gmail.com"
}
],
"engine": "gotpl",
"icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png",
"appVersion": "1.5.0",
"urls": [
"charts/harbor-0.2.1.tgz"
],
"created": "2018-08-29T10:26:29.625749155Z",
"digest": "2538edf4ddb797af8e025f3bd6226270440110bbdb689bad48656a519a154236"
},
{
"name": "harbor",
"home": "https://github.com/vmware/harbor",
"sources": [
"https://github.com/vmware/harbor/tree/master/contrib/helm/harbor"
],
"version": "0.2.0",
"description": "An Enterprise-class Docker Registry by VMware",
"keywords": [
"vmware",
"docker",
"registry",
"harbor"
],
"maintainers": [
{
"name": "Jesse Hu",
"email": "huh@vmware.com"
},
{
"name": "paulczar",
"email": "username.taken@gmail.com"
}
],
"engine": "gotpl",
"icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png",
"appVersion": "1.5.0",
"urls": [
"charts/harbor-0.2.0.tgz"
],
"created": "2018-08-29T10:26:21.141611102Z",
"digest": "fc8aae8dade9f0dfca12e9f1085081c49843d30a063a3fa7eb42497e3ceb277c"
}
]
`

61
src/chartserver/utils.go
View File

@ -4,8 +4,8 @@ import (
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"os"
"strings"
)
@ -14,30 +14,6 @@ const (
contentTypeJSON = "application/json"
)
// WriteError writes error to http client
func WriteError(w http.ResponseWriter, code int, err error) {
errorObj := make(map[string]string)
errorObj["error"] = err.Error()
errorContent, errorMarshal := json.Marshal(errorObj)
if errorMarshal != nil {
errorContent = []byte(err.Error())
}
w.WriteHeader(code)
w.Write(errorContent)
}
// WriteInternalError writes error with statusCode == 500
func WriteInternalError(w http.ResponseWriter, err error) {
WriteError(w, http.StatusInternalServerError, err)
}
// Write JSON data to http client
func writeJSONData(w http.ResponseWriter, data []byte) {
w.Header().Set(contentTypeHeader, contentTypeJSON)
w.WriteHeader(http.StatusOK)
w.Write(data)
}
// Extract error object '{"error": "****---***"}' from the content if existing
// nil error will be returned if it does exist
func extractError(content []byte) error {
@ -116,3 +92,38 @@ func parseRedisConfig(redisConfigV string) (string, error) {
return string(cfgData), nil
}
// What's the cache driver if it is set
func parseCacheDriver() (string, bool) {
driver, ok := os.LookupEnv(cacheDriverENVKey)
return strings.ToLower(driver), ok
}
// Get and parse the configuration for the chart cache
func getCacheConfig() (*ChartCacheConfig, error) {
driver, isSet := parseCacheDriver()
if !isSet {
return nil, nil
}
if driver != cacheDriverMem && driver != cacheDriverRedis {
return nil, fmt.Errorf("cache driver '%s' is not supported, only support 'memory' and 'redis'", driver)
}
if driver == cacheDriverMem {
return &ChartCacheConfig{
DriverType: driver,
}, nil
}
redisConfigV := os.Getenv(redisENVKey)
redisCfg, err := parseRedisConfig(redisConfigV)
if err != nil {
return nil, fmt.Errorf("failed to parse redis configurations from '%s' with error: %s", redisCfg, err)
}
return &ChartCacheConfig{
DriverType: driver,
Config: redisCfg,
}, nil
}

60
src/chartserver/utils_test.go
View File

@ -1,6 +1,8 @@
package chartserver
import (
"encoding/json"
"os"
"strings"
"testing"
)
@ -40,3 +42,61 @@ func TestParseRedisConfig(t *testing.T) {
}
}
}
func TestGetCacheConfig(t *testing.T) {
// case 1: no cache set
cacheConf, err := getCacheConfig()
if err != nil || cacheConf != nil {
t.Fatal("expect nil cache config and nil error but got non-nil one when parsing empty cache settings")
}
// case 2: unknown cache type
os.Setenv(cacheDriverENVKey, "unknown")
_, err = getCacheConfig()
if err == nil {
t.Fatal("expect non-nil error but got nil one when parsing unknown cache type")
}
// case 3: in memory cache type
os.Setenv(cacheDriverENVKey, cacheDriverMem)
memCacheConf, err := getCacheConfig()
if err != nil || memCacheConf == nil || memCacheConf.DriverType != cacheDriverMem {
t.Fatal("expect in memory cache driver but got invalid one")
}
// case 4: wrong redis cache conf
os.Setenv(cacheDriverENVKey, cacheDriverRedis)
os.Setenv(redisENVKey, "")
_, err = getCacheConfig()
if err == nil {
t.Fatal("expect non-nil error but got nil one when parsing a invalid redis cache conf")
}
// case 5: redis cache conf
os.Setenv(redisENVKey, "redis:6379,100,Passw0rd,1")
redisConf, err := getCacheConfig()
if err != nil {
t.Fatalf("expect nil error but got non-nil one when parsing valid redis conf")
}
if redisConf == nil || redisConf.DriverType != cacheDriverRedis {
t.Fatal("expect redis cache driver but got invalid one")
}
conf := make(map[string]string)
if err = json.Unmarshal([]byte(redisConf.Config), &conf); err != nil {
t.Fatal(err)
}
if v, ok := conf["conn"]; !ok {
t.Fatal("expect 'conn' filed in the parsed conf but got nothing")
} else {
if v != "redis:6379" {
t.Fatalf("expect %s but got %s", "redis:6379", v)
}
}
// clear
os.Unsetenv(cacheDriverENVKey)
os.Unsetenv(redisENVKey)
}

25
src/core/api/api_test.go
View File

@ -19,11 +19,13 @@ import (
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strconv"
"strings"
"testing"
"github.com/goharbor/harbor/src/chartserver"
"github.com/goharbor/harbor/src/common"
"github.com/astaxie/beego"
@ -32,6 +34,7 @@ import (
"github.com/goharbor/harbor/src/common/dao/project"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/models"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -292,3 +295,25 @@ func clean() {
}
}
}
// Provides a mock chart controller for deletable test cases
func mockChartController() (*httptest.Server, *chartserver.Controller, error) {
mockServer := httptest.NewServer(htesting.MockChartRepoHandler)
var oldController, newController *chartserver.Controller
url, err := url.Parse(mockServer.URL)
if err == nil {
newController, err = chartserver.NewController(url)
}
if err != nil {
mockServer.Close()
return nil, nil, err
}
// Override current controller and keep the old one for restoring
oldController = chartController
chartController = newController
return mockServer, oldController, nil
}

25
src/core/api/base.go
View File

@ -17,6 +17,7 @@ package api
import (
"net/http"
yaml "github.com/ghodss/yaml"
"github.com/goharbor/harbor/src/common/api"
"github.com/goharbor/harbor/src/common/security"
"github.com/goharbor/harbor/src/common/utils/log"
@ -26,6 +27,10 @@ import (
"github.com/goharbor/harbor/src/core/utils"
)
const (
yamlFileContentType = "application/x-yaml"
)
// BaseController ...
type BaseController struct {
api.BaseAPI
@ -98,6 +103,26 @@ func (b *BaseController) SendForbiddenError(err error) {
b.RenderFormatedError(http.StatusForbidden, err)
}
// WriteJSONData writes the JSON data to the client.
func (b *BaseController) WriteJSONData(object interface{}) {
b.Data["json"] = object
b.ServeJSON()
}
// WriteYamlData writes the yaml data to the client.
func (b *BaseController) WriteYamlData(object interface{}) {
yData, err := yaml.Marshal(object)
if err != nil {
b.SendInternalServerError(err)
return
}
w := b.Ctx.ResponseWriter
w.Header().Set("Content-Type", yamlFileContentType)
w.WriteHeader(http.StatusOK)
w.Write(yData)
}
// Init related objects/configurations for the API controllers
func Init() error {
// If chart repository is not enabled then directly return

2
src/core/api/chart_label.go
View File

@ -55,7 +55,7 @@ func (cla *ChartLabelAPI) Prepare() {
chartName := cla.GetStringFromPath(nameParam)
version := cla.GetStringFromPath(versionParam)
if _, err = chartController.GetUtilityHandler().GetChartVersion(project, chartName, version); err != nil {
if _, err = chartController.GetChartVersion(project, chartName, version); err != nil {
cla.SendNotFoundError(err)
return
}

201
src/core/api/chart_repository.go
View File

@ -2,7 +2,6 @@ package api
import (
"bytes"
"context"
"errors"
"fmt"
"io"
@ -12,6 +11,9 @@ import (
"net/url"
"strings"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/core/label"
"github.com/goharbor/harbor/src/chartserver"
hlog "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
@ -45,6 +47,9 @@ type ChartRepositoryAPI struct {
// The base controller to provide common utilities
BaseController
// For label management
labelManager *label.BaseManager
// Keep the namespace if existing
namespace string
}
@ -62,7 +67,7 @@ func (cra *ChartRepositoryAPI) Prepare() {
// Exclude the following URI
// -/index.yaml
// -/api/chartserver/health
incomingURI := cra.Ctx.Request.RequestURI
incomingURI := cra.Ctx.Request.URL.Path
if incomingURI == rootUploadingEndpoint {
// Forward to the default repository
cra.namespace = defaultRepo
@ -75,18 +80,19 @@ func (cra *ChartRepositoryAPI) Prepare() {
}
}
// Rewrite URL path
cra.rewriteURLPath(cra.Ctx.Request)
// Init label manager
cra.labelManager = &label.BaseManager{}
}
// GetHealthStatus handles GET /api/chartserver/health
// GetHealthStatus handles GET /api/chartrepo/health
func (cra *ChartRepositoryAPI) GetHealthStatus() {
// Check access
if !cra.requireAccess(cra.namespace, accessLevelSystem) {
return
}
chartController.GetBaseHandler().GetHealthStatus(cra.Ctx.ResponseWriter, cra.Ctx.Request)
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// GetIndexByRepo handles GET /:repo/index.yaml
@ -96,7 +102,8 @@ func (cra *ChartRepositoryAPI) GetIndexByRepo() {
return
}
chartController.GetRepositoryHandler().GetIndexFileWithNS(cra.Ctx.ResponseWriter, cra.Ctx.Request)
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// GetIndex handles GET /index.yaml
@ -106,7 +113,24 @@ func (cra *ChartRepositoryAPI) GetIndex() {
return
}
chartController.GetRepositoryHandler().GetIndexFile(cra.Ctx.ResponseWriter, cra.Ctx.Request)
results, err := cra.ProjectMgr.List(nil)
if err != nil {
cra.SendInternalServerError(err)
return
}
namespaces := []string{}
for _, r := range results.Projects {
namespaces = append(namespaces, r.Name)
}
indexFile, err := chartController.GetIndexFile(namespaces)
if err != nil {
cra.SendInternalServerError(err)
return
}
cra.WriteYamlData(indexFile)
}
// DownloadChart handles GET /:repo/charts/:filename
@ -116,7 +140,8 @@ func (cra *ChartRepositoryAPI) DownloadChart() {
return
}
chartController.GetRepositoryHandler().DownloadChartObject(cra.Ctx.ResponseWriter, cra.Ctx.Request)
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// ListCharts handles GET /api/:repo/charts
@ -126,7 +151,13 @@ func (cra *ChartRepositoryAPI) ListCharts() {
return
}
chartController.GetManipulationHandler().ListCharts(cra.Ctx.ResponseWriter, cra.Ctx.Request)
charts, err := chartController.ListCharts(cra.namespace)
if err != nil {
cra.SendInternalServerError(err)
return
}
cra.WriteJSONData(charts)
}
// ListChartVersions GET /api/:repo/charts/:name
@ -136,7 +167,25 @@ func (cra *ChartRepositoryAPI) ListChartVersions() {
return
}
chartController.GetManipulationHandler().GetChart(cra.Ctx.ResponseWriter, cra.Ctx.Request)
chartName := cra.GetStringFromPath(nameParam)
versions, err := chartController.GetChart(cra.namespace, chartName)
if err != nil {
cra.SendInternalServerError(err)
return
}
// Append labels
for _, chartVersion := range versions {
labels, err := cra.labelManager.GetLabelsOfResource(common.ResourceTypeChart, chartFullName(cra.namespace, chartVersion.Name, chartVersion.Version))
if err != nil {
cra.SendInternalServerError(err)
return
}
chartVersion.Labels = labels
}
cra.WriteJSONData(versions)
}
// GetChartVersion handles GET /api/:repo/charts/:name/:version
@ -146,11 +195,25 @@ func (cra *ChartRepositoryAPI) GetChartVersion() {
return
}
// Let's pass the namespace via the context of request
req := cra.Ctx.Request
*req = *(req.WithContext(context.WithValue(req.Context(), chartserver.NamespaceContextKey, cra.namespace)))
// Get other parameters
chartName := cra.GetStringFromPath(nameParam)
version := cra.GetStringFromPath(versionParam)
chartController.GetManipulationHandler().GetChartVersion(cra.Ctx.ResponseWriter, req)
chartVersion, err := chartController.GetChartVersionDetails(cra.namespace, chartName, version)
if err != nil {
cra.SendInternalServerError(err)
return
}
// Append labels
labels, err := cra.labelManager.GetLabelsOfResource(common.ResourceTypeChart, chartFullName(cra.namespace, chartName, version))
if err != nil {
cra.SendInternalServerError(err)
return
}
chartVersion.Labels = labels
cra.WriteJSONData(chartVersion)
}
// DeleteChartVersion handles DELETE /api/:repo/charts/:name/:version
@ -160,7 +223,20 @@ func (cra *ChartRepositoryAPI) DeleteChartVersion() {
return
}
chartController.GetManipulationHandler().DeleteChartVersion(cra.Ctx.ResponseWriter, cra.Ctx.Request)
// Get other parameters
chartName := cra.GetStringFromPath(nameParam)
version := cra.GetStringFromPath(versionParam)
// Try to remove labels from deleting chart if exitsing
if err := cra.removeLabelsFromChart(chartName, version); err != nil {
cra.SendInternalServerError(err)
return
}
if err := chartController.DeleteChartVersion(cra.namespace, chartName, version); err != nil {
cra.SendInternalServerError(err)
return
}
}
// UploadChartVersion handles POST /api/:repo/charts
@ -184,12 +260,13 @@ func (cra *ChartRepositoryAPI) UploadChartVersion() {
formField: formFiledNameForProv,
})
if err := cra.rewriteFileContent(formFiles, cra.Ctx.Request); err != nil {
chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err)
cra.SendInternalServerError(err)
return
}
}
chartController.GetManipulationHandler().UploadChartVersion(cra.Ctx.ResponseWriter, cra.Ctx.Request)
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// UploadChartProvFile handles POST /api/:repo/prov
@ -208,12 +285,13 @@ func (cra *ChartRepositoryAPI) UploadChartProvFile() {
mustHave: true,
})
if err := cra.rewriteFileContent(formFiles, cra.Ctx.Request); err != nil {
chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err)
cra.SendInternalServerError(err)
return
}
}
chartController.GetManipulationHandler().UploadProvenanceFile(cra.Ctx.ResponseWriter, cra.Ctx.Request)
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// DeleteChart deletes all the chart versions of the specified chart.
@ -226,44 +304,39 @@ func (cra *ChartRepositoryAPI) DeleteChart() {
// Get other parameters from the request
chartName := cra.GetStringFromPath(nameParam)
if err := chartController.GetUtilityHandler().DeleteChart(cra.namespace, chartName); err != nil {
chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err)
// Remove labels from all the deleting chart versions under the chart
chartVersions, err := chartController.GetChart(cra.namespace, chartName)
if err != nil {
cra.SendInternalServerError(err)
return
}
for _, chartVersion := range chartVersions {
if err := cra.removeLabelsFromChart(chartName, chartVersion.GetVersion()); err != nil {
cra.SendInternalServerError(err)
return
}
}
if err := chartController.DeleteChart(cra.namespace, chartName); err != nil {
cra.SendInternalServerError(err)
return
}
}
// Rewrite the incoming URL with the right backend URL pattern
// Remove 'chartrepo' from the endpoints of manipulation API
// Remove 'chartrepo' from the endpoints of repository services
func (cra *ChartRepositoryAPI) rewriteURLPath(req *http.Request) {
incomingURLPath := req.RequestURI
defer func() {
hlog.Debugf("Incoming URL '%s' is rewritten to '%s'", incomingURLPath, req.URL.String())
}()
// Health check endpoint
if incomingURLPath == chartRepoHealthEndpoint {
req.URL.Path = "/health"
return
func (cra *ChartRepositoryAPI) removeLabelsFromChart(chartName, version string) error {
// Try to remove labels from deleting chart if exitsing
resourceID := chartFullName(cra.namespace, chartName, version)
labels, err := cra.labelManager.GetLabelsOfResource(common.ResourceTypeChart, resourceID)
if err == nil && len(labels) > 0 {
for _, l := range labels {
if err := cra.labelManager.RemoveLabelFromResource(common.ResourceTypeChart, resourceID, l.ID); err != nil {
return err
}
}
}
// Root uploading endpoint
if incomingURLPath == rootUploadingEndpoint {
req.URL.Path = strings.Replace(incomingURLPath, "chartrepo", defaultRepo, 1)
return
}
// Repository endpoints
if strings.HasPrefix(incomingURLPath, "/chartrepo") {
req.URL.Path = strings.TrimPrefix(incomingURLPath, "/chartrepo")
return
}
// API endpoints
if strings.HasPrefix(incomingURLPath, "/api/chartrepo") {
req.URL.Path = strings.Replace(incomingURLPath, "/chartrepo", "", 1)
return
}
return nil
}
// Check if there exists a valid namespace
@ -272,20 +345,20 @@ func (cra *ChartRepositoryAPI) rewriteURLPath(req *http.Request) {
func (cra *ChartRepositoryAPI) requireNamespace(namespace string) bool {
// Actually, never should be like this
if len(namespace) == 0 {
cra.HandleBadRequest(":repo should be in the request URL")
cra.SendBadRequestError(errors.New(":repo should be in the request URL"))
return false
}
existsing, err := cra.ProjectMgr.Exists(namespace)
if err != nil {
// Check failed with error
cra.renderError(http.StatusInternalServerError, fmt.Sprintf("failed to check existence of namespace %s with error: %s", namespace, err.Error()))
cra.SendInternalServerError(fmt.Errorf("failed to check existence of namespace %s with error: %s", namespace, err.Error()))
return false
}
// Not existing
if !existsing {
cra.renderError(http.StatusBadRequest, fmt.Sprintf("namespace %s is not existing", namespace))
cra.SendBadRequestError(fmt.Errorf("namespace %s is not existing", namespace))
return false
}
@ -328,7 +401,7 @@ func (cra *ChartRepositoryAPI) requireAccess(namespace string, accessLevel uint)
}
default:
// access rejected for invalid scope
cra.renderError(http.StatusForbidden, "unrecognized access scope")
cra.SendForbiddenError(errors.New("unrecognized access scope"))
return false
}
@ -336,23 +409,18 @@ func (cra *ChartRepositoryAPI) requireAccess(namespace string, accessLevel uint)
if err != nil {
// Unauthenticated, return 401
if !cra.SecurityCtx.IsAuthenticated() {
cra.renderError(http.StatusUnauthorized, "Unauthorized")
cra.SendUnAuthorizedError(errors.New("Unauthorized"))
return false
}
// Authenticated, return 403
cra.renderError(http.StatusForbidden, err.Error())
cra.SendForbiddenError(err)
return false
}
return true
}
// write error message with unified format
func (cra *ChartRepositoryAPI) renderError(code int, text string) {
chartserver.WriteError(cra.Ctx.ResponseWriter, code, errors.New(text))
}
// formFile is used to represent the uploaded files in the form
type formFile struct {
// form field key contains the form file
@ -442,3 +510,8 @@ func initializeChartController() (*chartserver.Controller, error) {
func isMultipartFormData(req *http.Request) bool {
return strings.Contains(req.Header.Get(headerContentType), contentTypeMultipart)
}
// Return the chart full name
func chartFullName(namespace, chartName, version string) string {
return fmt.Sprintf("%s/%s:%s", namespace, chartName, version)
}

200
src/core/api/chart_repository_test.go
View File

@ -3,51 +3,18 @@ package api
import (
"errors"
"net/http"
"net/http/httptest"
"testing"
"github.com/goharbor/harbor/src/chartserver"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/core/promgr/metamgr"
)
// Test the URL rewrite function
func TestURLRewrite(t *testing.T) {
chartAPI := &ChartRepositoryAPI{}
req, err := createRequest(http.MethodGet, "/api/chartrepo/health")
if err != nil {
t.Fatal(err)
}
chartAPI.rewriteURLPath(req)
if req.URL.Path != "/health" {
t.Fatalf("Expect url format %s but got %s", "/health", req.URL.Path)
}
req, err = createRequest(http.MethodGet, "/api/chartrepo/library/charts")
if err != nil {
t.Fatal(err)
}
chartAPI.rewriteURLPath(req)
if req.URL.Path != "/api/library/charts" {
t.Fatalf("Expect url format %s but got %s", "/api/library/charts", req.URL.Path)
}
req, err = createRequest(http.MethodPost, "/api/chartrepo/charts")
if err != nil {
t.Fatal(err)
}
chartAPI.rewriteURLPath(req)
if req.URL.Path != "/api/library/charts" {
t.Fatalf("Expect url format %s but got %s", "/api/library/charts", req.URL.Path)
}
req, err = createRequest(http.MethodGet, "/chartrepo/library/index.yaml")
if err != nil {
t.Fatal(err)
}
chartAPI.rewriteURLPath(req)
if req.URL.Path != "/library/index.yaml" {
t.Fatalf("Expect url format %s but got %s", "/library/index.yaml", req.URL.Path)
}
}
var (
crOldController *chartserver.Controller
crMockServer *httptest.Server
)
// Test access checking
func TestRequireAccess(t *testing.T) {
@ -98,6 +65,157 @@ func TestRequireNamespace(t *testing.T) {
}
}
// Prepare
func TestPrepareEnv(t *testing.T) {
var err error
crMockServer, crOldController, err = mockChartController()
if err != nil {
t.Fatalf("Failed to start mock chart service with error: %s", err)
}
}
// Test get health
func TestGetHealthStatus(t *testing.T) {
status := make(map[string]interface{})
err := handleAndParse(&testingRequest{
url: "/api/chartrepo/health",
method: http.MethodGet,
credential: sysAdmin,
}, &status)
if err != nil {
t.Fatal(err)
}
if _, ok := status["health"]; !ok {
t.Fatal("expect 'health' but got nil")
}
}
// Test get index by repo
func TestGetIndexByRepo(t *testing.T) {
runCodeCheckingCases(t, &codeCheckingCase{
request: &testingRequest{
url: "/chartrepo/library/index.yaml",
method: http.MethodGet,
credential: projDeveloper,
},
code: http.StatusOK,
})
}
// Test get index
func TestGetIndex(t *testing.T) {
runCodeCheckingCases(t, &codeCheckingCase{
request: &testingRequest{
url: "/chartrepo/index.yaml",
method: http.MethodGet,
credential: sysAdmin,
},
code: http.StatusOK,
})
}
// Test download chart
func TestDownloadChart(t *testing.T) {
runCodeCheckingCases(t, &codeCheckingCase{
request: &testingRequest{
url: "/chartrepo/library/charts/harbor-0.2.0.tgz",
method: http.MethodGet,
credential: projDeveloper,
},
code: http.StatusOK,
})
}
// Test get charts
func TesListCharts(t *testing.T) {
charts := make([]*chartserver.ChartInfo, 0)
err := handleAndParse(&testingRequest{
url: "/api/chartrepo/library/charts",
method: http.MethodGet,
credential: projAdmin,
}, &charts)
if err != nil {
t.Fatal(err)
}
if len(charts) != 2 {
t.Fatalf("expect 2 charts but got %d", len(charts))
}
}
// Test get chart versions
func TestListChartVersions(t *testing.T) {
chartVersions := make(chartserver.ChartVersions, 0)
err := handleAndParse(&testingRequest{
url: "/api/chartrepo/library/charts/harbor",
method: http.MethodGet,
credential: projAdmin,
}, &chartVersions)
if err != nil {
t.Fatal(err)
}
if len(chartVersions) != 2 {
t.Fatalf("expect 2 chart versions but got %d", len(chartVersions))
}
}
// Test get chart version details
func TestGetChartVersion(t *testing.T) {
chartV := &chartserver.ChartVersionDetails{}
err := handleAndParse(&testingRequest{
url: "/api/chartrepo/library/charts/harbor/0.2.0",
method: http.MethodGet,
credential: projAdmin,
}, chartV)
if err != nil {
t.Fatal(err)
}
if chartV.Metadata.GetName() != "harbor" {
t.Fatalf("expect get chart 'harbor' but got %s", chartV.Metadata.GetName())
}
if chartV.Metadata.GetVersion() != "0.2.0" {
t.Fatalf("expect get chart version '0.2.0' but got %s", chartV.Metadata.GetVersion())
}
}
// Test delete chart version
func TestDeleteChartVersion(t *testing.T) {
runCodeCheckingCases(t, &codeCheckingCase{
request: &testingRequest{
url: "/api/chartrepo/library/charts/harbor/0.2.1",
method: http.MethodDelete,
credential: projAdmin,
},
code: http.StatusOK,
})
}
// Test delete chart
func TestDeleteChart(t *testing.T) {
runCodeCheckingCases(t, &codeCheckingCase{
request: &testingRequest{
url: "/api/chartrepo/library/charts/harbor",
method: http.MethodDelete,
credential: projDeveloper,
},
code: http.StatusOK,
})
}
// Clear
func TestClearEnv(t *testing.T) {
crMockServer.Close()
chartController = crOldController
}
func createRequest(method string, url string) (*http.Request, error) {
req, err := http.NewRequest(method, url, nil)
if err != nil {
@ -132,7 +250,7 @@ func (mpm *mockProjectManager) List(query *models.ProjectQueryParam) (*models.Pr
Projects: make([]*models.Project, 0),
}
results.Projects = append(results.Projects, &models.Project{ProjectID: 0, Name: "repo1"})
results.Projects = append(results.Projects, &models.Project{ProjectID: 0, Name: "library"})
results.Projects = append(results.Projects, &models.Project{ProjectID: 1, Name: "repo2"})
return results, nil
@ -228,7 +346,7 @@ func (msc *mockSecurityContext) HasAllPerm(projectIDOrName interface{}) bool {
// Get current user's all project
func (msc *mockSecurityContext) GetMyProjects() ([]*models.Project, error) {
return []*models.Project{{ProjectID: 0, Name: "repo1"}}, nil
return []*models.Project{{ProjectID: 0, Name: "library"}}, nil
}
// Get user's role in provided project

17
src/core/api/harborapi_test.go
View File

@ -170,6 +170,23 @@ func init() {
beego.Router("/api/system/gc/:id", &GCAPI{}, "get:GetGC")
beego.Router("/api/system/gc/:id([0-9]+)/log", &GCAPI{}, "get:GetLog")
beego.Router("/api/system/gc/schedule", &GCAPI{}, "get:Get;put:Put;post:Post")
// Charts are controlled under projects
chartRepositoryAPIType := &ChartRepositoryAPI{}
beego.Router("/api/chartrepo/health", chartRepositoryAPIType, "get:GetHealthStatus")
beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "get:ListCharts")
beego.Router("/api/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "get:ListChartVersions")
beego.Router("/api/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "delete:DeleteChart")
beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "get:GetChartVersion")
beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "delete:DeleteChartVersion")
beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
beego.Router("/api/chartrepo/:repo/prov", chartRepositoryAPIType, "post:UploadChartProvFile")
beego.Router("/api/chartrepo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
// Repository services
beego.Router("/chartrepo/:repo/index.yaml", chartRepositoryAPIType, "get:GetIndexByRepo")
beego.Router("/chartrepo/index.yaml", chartRepositoryAPIType, "get:GetIndex")
beego.Router("/chartrepo/:repo/charts/:filename", chartRepositoryAPIType, "get:DownloadChart")
// Labels for chart
chartLabelAPIType := &ChartLabelAPI{}
beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels", chartLabelAPIType, "get:GetLabels;post:MarkLabel")

2
src/core/api/project.go
View File

@ -298,7 +298,7 @@ func (p *ProjectAPI) deletable(projectID int64) (*deletableResp, error) {
// Check helm charts number
if config.WithChartMuseum() {
charts, err := chartController.GetUtilityHandler().GetChartsByNs(p.project.Name)
charts, err := chartController.ListCharts(p.project.Name)
if err != nil {
return nil, err
}

78
src/core/api/project_test.go
View File

@ -16,14 +16,10 @@ package api
import (
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"testing"
"time"
"github.com/goharbor/harbor/src/chartserver"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/tests/apitests/apilib"
@ -409,77 +405,3 @@ func TestDeletable(t *testing.T) {
assert.Equal(t, http.StatusOK, code)
assert.False(t, del)
}
// Provides a mock chart controller for deletable test cases
func mockChartController() (*httptest.Server, *chartserver.Controller, error) {
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.RequestURI {
case "/api/project_for_test_deletable/charts":
if r.Method == http.MethodGet {
w.Write([]byte("{}"))
return
}
case "/api/library/charts/harbor/0.2.0":
if r.Method == http.MethodGet {
w.Write([]byte(chartVersionOfHarbor020))
return
}
}
w.WriteHeader(http.StatusNotImplemented)
w.Write([]byte("not supported"))
}))
var oldController, newController *chartserver.Controller
url, err := url.Parse(mockServer.URL)
if err == nil {
newController, err = chartserver.NewController(url)
}
if err != nil {
mockServer.Close()
return nil, nil, err
}
// Override current controller and keep the old one for restoring
oldController = chartController
chartController = newController
return mockServer, oldController, nil
}
var chartVersionOfHarbor020 = `
{
"name": "harbor",
"home": "https://github.com/vmware/harbor",
"sources": [
"https://github.com/vmware/harbor/tree/master/contrib/helm/harbor"
],
"version": "0.2.0",
"description": "An Enterprise-class Docker Registry by VMware",
"keywords": [
"vmware",
"docker",
"registry",
"harbor"
],
"maintainers": [
{
"name": "Jesse Hu",
"email": "huh@vmware.com"
},
{
"name": "paulczar",
"email": "username.taken@gmail.com"
}
],
"engine": "gotpl",
"icon": "https://raw.githubusercontent.com/vmware/harbor/master/docs/img/harbor_logo.png",
"appVersion": "1.5.0",
"urls": [
"charts/harbor-0.2.0.tgz"
],
"created": "2018-08-29T10:26:21.141611102Z",
"digest": "fc8aae8dade9f0dfca12e9f1085081c49843d30a063a3fa7eb42497e3ceb277c"
}
`

2
src/core/api/search.go
View File

@ -125,7 +125,7 @@ func (s *SearchAPI) Get() {
}
if searchHandler == nil {
searchHandler = chartController.GetRepositoryHandler().SearchChart
searchHandler = chartController.SearchChart
}
chartResults, err := searchHandler(keyword, proNames)

326
src/testing/chart_utility.go Normal file
View File

File diff suppressed because one or more lines are too long

2
tests/coverage4gotest.sh
View File

@ -20,7 +20,7 @@ function listDeps(){
done
}
packages=$(go list ./... | grep -v -E 'vendor|tests')
packages=$(go list ./... | grep -v -E 'vendor|tests|testing')
for package in $packages
do