Keep the chart server related configurations in adminserver

append chart server related config options to the supporting list of adminserver
provide chart server related config access method in the API layer
update prepare script and ui env template file to enable cache driver config for chart server API
append flag info in the systeminfo API to indicate if chart server is deployed with Harbor
refactor the response rewriting logic to return structual error object
add api init method to initilizing objects required in API handlers
chage owner of the storage folder
update offline/online package scripts in Harbor-Util.robot

.travis.yml

@@ -28,7 +28,6 @@ env:
   SELF_REGISTRATION: on
   KEY_PATH: /data/secretkey
   REDIS_HOST: localhost
-  BACKEND_CHART_SERVER: http://localhost:9090
 
 before_install:
    - sudo ./tests/hostcfg.sh

Makefile

@@ -312,7 +312,8 @@ build:
 	make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) -e MARIADBVERSION=$(MARIADBVERSION) \
 	 -e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) \
 	 -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \
-	 -e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION)
+	 -e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \
+	 -e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER)
 
 modify_composefile: modify_composefile_notary modify_composefile_clair modify_composefile_chartmuseum
 	@echo "preparing docker-compose file..."
@@ -444,7 +445,7 @@ start:
 	@echo "Start complete. You can visit harbor now."
 
 down:
-	@echo "Please make sure to set -e NOTARYFLAG=true/CLAIRFLAG=true if you are using Notary/CLAIR in Harbor, otherwise the Notary/CLAIR containers cannot be stop automaticlly."
+	@echo "Please make sure to set -e NOTARYFLAG=true/CLAIRFLAG=true/CHARTFLAG=true if you are using Notary/CLAIR/Chartmuseum in Harbor, otherwise the Notary/CLAIR/Chartmuseum containers cannot be stop automaticlly."
 	@while [ -z "$$CONTINUE" ]; do \
         read -r -p "Type anything but Y or y to exit. [Y/N]: " CONTINUE; \
     done ; \

make/common/templates/adminserver/env

@@ -61,6 +61,7 @@ REGISTRY_STORAGE_PROVIDER_NAME=$storage_provider_name
 READ_ONLY=false
 SKIP_RELOAD_ENV_PATTERN=$skip_reload_env_pattern
 RELOAD_KEY=$reload_key
-CHART_SERVICE_URL=$chart_service_url
+CHART_REPOSITORY_URL=$chart_repository_url
 LDAP_GROUP_ADMIN_DN=$ldap_group_admin_dn
 REGISTRY_CONTROLLER_URL=$registry_controller_url
+WITH_CHARTMUSEUM=$with_chartmuseum

make/common/templates/chartserver/env

@@ -8,7 +8,7 @@ CACHE_REDIS_PASSWORD=$cache_redis_password
 CACHE_REDIS_DB=$cache_redis_db_index
 
 # Credential for internal communication
-BASIC_AUTH_USER="chart_controller"
+BASIC_AUTH_USER=chart_controller
 BASIC_AUTH_PASS=$ui_secret
 
 # Multiple tenants
@@ -28,14 +28,14 @@ DISABLE_METRICS=false
 DISABLE_API=false
 DISABLE_STATEFILES=false
 ALLOW_OVERWRITE=false
-CHART_URL=""
+CHART_URL=
 AUTH_ANONYMOUS_GET=false
-TLS_CERT=""
-TLS_KEY=""
-CONTEXT_PATH=""
+TLS_CERT=
+TLS_KEY=
+CONTEXT_PATH=
 INDEX_LIMIT=0
 MAX_STORAGE_OBJECTS=0
 MAX_UPLOAD_SIZE=20971520
-CHART_POST_FORM_FIELD_NAME="chart"
-PROV_POST_FORM_FIELD_NAME="prov"
+CHART_POST_FORM_FIELD_NAME=chart
+PROV_POST_FORM_FIELD_NAME=prov
 

make/common/templates/ui/env

@@ -7,3 +7,4 @@ ADMINSERVER_URL=$adminserver_url
 UAA_CA_ROOT=/etc/ui/certificates/uaa_ca.pem
 _REDIS_URL=$redis_url
 SYNC_REGISTRY=false
+CHART_CACHE_DRIVER=$chart_cache_driver

make/docker-compose.chartmuseum.tpl

@@ -14,7 +14,6 @@ services:
     container_name: chartmuseum
     image: vmware/chartmuseum-photon:__chartmuseum_version__
     restart: always
-    cpu_quota: 150000
     networks:
       - harbor-chartmuseum
     depends_on:

make/photon/chartserver/Dockerfile

@@ -1,5 +1,4 @@
-FROM vmware/photon:1.0
-
+FROM photon:1.0
 
 RUN tdnf distro-sync -y \
     && tdnf erase vim -y \

make/photon/chartserver/docker-entrypoint.sh

@@ -1,4 +1,12 @@
 #!/bin/bash
 set -e
+
+#/chart_storage is the directory in the contaienr for storing the chart artifacts
+#if storage driver is set to 'local'
+if [ -d /chart_storage ]; then
+    chown 10000:10000 -R /chart_storage
+fi
+
+# Start the server process
 sudo -E -H -u \#10000 sh -c "/chartserver/chartm" #Parameters are set by ENV
 set +e

make/prepare

@@ -333,7 +333,7 @@ token_service_url = "http://ui:8080/service/token"
 jobservice_url = "http://jobservice:8080"
 clair_url = "http://clair:6060"
 notary_url = "http://notary-server:4443"
-chart_service_url = "http://chartmuseum:9999"
+chart_repository_url = "http://chartmuseum:9999"
 
 if len(admiral_url) != 0  and admiral_url != "NA":
 #VIC overwrites the data volume path, which by default should be same as the value of secretkey_path
@@ -421,16 +421,24 @@ render(os.path.join(templates_dir, "adminserver", "env"),
         notary_url=notary_url,
         reload_key=reload_key,
         skip_reload_env_pattern=skip_reload_env_pattern,
-        chart_service_url=chart_service_url
-        registry_controller_url = registry_controller_url
+        chart_repository_url=chart_repository_url,
+        registry_controller_url = registry_controller_url,
+        with_chartmuseum=args.chart_mode
 	)
 
+# set cache for chart repo server
+# default set 'memory' mode, if redis is configured then set to 'redis'
+chart_cache_driver = "memory"
+if len(redis_url) > 0:
+    chart_cache_driver = "redis"
+
 render(os.path.join(templates_dir, "ui", "env"), 
         ui_conf_env, 
         ui_secret=ui_secret,
         jobservice_secret=jobservice_secret,
         redis_url = redis_url,
-        adminserver_url = adminserver_url
+        adminserver_url = adminserver_url,
+        chart_cache_driver = chart_cache_driver
         )
 
 registry_config_file_ha = "config_ha.yml"
@@ -651,12 +659,12 @@ if args.ha_mode:
 # config chart repository
 if args.chart_mode:
     chartm_temp_dir = os.path.join(templates_dir, "chartserver")
-    chrtm_config_dir = os.path.join(config_dir, "chartserver")
+    chartm_config_dir = os.path.join(config_dir, "chartserver")
     chartm_env = os.path.join(config_dir, "chartserver", "env")
 
-    if not os.path.isdir(chrtm_config_dir):
-        print ("Create config folder: %s" % chrtm_config_dir)
-        os.makedirs(chrtm_config_dir)
+    if not os.path.isdir(chartm_config_dir):
+        print ("Create config folder: %s" % chartm_config_dir)
+        os.makedirs(chartm_config_dir)
 
     # process redis info
     cache_store = ""
@@ -733,7 +741,7 @@ if args.chart_mode:
 
     render(os.path.join(chartm_temp_dir, "env"),
         chartm_env,
-        cache_store=storage_driver,
+        cache_store=cache_store,
         cache_redis_addr=cache_redis_addr,
         cache_redis_password=cache_redis_password,
         cache_redis_db_index=cache_redis_db_index,

src/adminserver/systemcfg/store/database/driver_db.go

@@ -49,6 +49,7 @@ var (
 		common.LDAPVerifyCert:   true,
 		common.UAAVerifyCert:    true,
 		common.ReadOnly:         true,
+		common.WithChartMuseum:  true,
 	}
 	mapKeys = map[string]bool{
 		common.ScanAllPolicy: true,

src/adminserver/systemcfg/systemcfg.go

@@ -166,6 +166,11 @@ var (
 		},
 		common.ReloadKey:        "RELOAD_KEY",
 		common.LdapGroupAdminDn: "LDAP_GROUP_ADMIN_DN",
+		common.ChartRepoURL:     "CHART_REPOSITORY_URL",
+		common.WithChartMuseum: &parser{
+			env:   "WITH_CHARTMUSEUM",
+			parse: parseStringToBool,
+		},
 	}
 
 	// configurations need read from environment variables
@@ -220,6 +225,11 @@ var (
 		common.ClairURL:                    "CLAIR_URL",
 		common.NotaryURL:                   "NOTARY_URL",
 		common.DatabaseType:                "DATABASE_TYPE",
+		common.ChartRepoURL:                "CHART_REPOSITORY_URL",
+		common.WithChartMuseum: &parser{
+			env:   "WITH_CHARTMUSEUM",
+			parse: parseStringToBool,
+		},
 	}
 )
 

src/chartserver/controller_test.go

@@ -179,7 +179,7 @@ func TestResponseRewrite(t *testing.T) {
 	if msg, ok := errObj["error"]; !ok {
 		t.Fatal("Expect an error message from server but got nothing")
 	} else {
-		if !strings.Contains(msg.(string), "operation request from unauthentic source is rejected") {
+		if !strings.Contains(msg.(string), "operation request from unauthorized source is rejected") {
 			t.Fatal("Missing the required error message")
 		}
 	}

src/chartserver/reverse_proxy.go

@@ -3,6 +3,7 @@ package chartserver
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -73,25 +74,41 @@ func director(target *url.URL, cred *Credential, req *http.Request) {
 
 //Modify the http response
 func modifyResponse(res *http.Response) error {
-	//Detect the 401 code, if it is,
-	//overwrite it to 500.
-	//We also re-write the error content
-	if res.StatusCode == http.StatusUnauthorized {
-		errorObj := make(map[string]string)
-		errorObj["error"] = "operation request from unauthentic source is rejected"
-		content, err := json.Marshal(errorObj)
-		if err != nil {
-			return err
-		}
-
-		size := len(content)
-		body := ioutil.NopCloser(bytes.NewReader(content))
-		res.Body = body
-		res.ContentLength = int64(size)
-		res.Header.Set(contentLengthHeader, strconv.Itoa(size))
-		res.StatusCode = http.StatusInternalServerError
+	//Accept cases
+	//Success or redirect
+	if res.StatusCode >= http.StatusOK && res.StatusCode <= http.StatusTemporaryRedirect {
+		return nil
 	}
 
+	//Detect the 401 code, if it is,overwrite it to 500.
+	//We also re-write the error content to structural error object
+	errorObj := make(map[string]string)
+	if res.StatusCode == http.StatusUnauthorized {
+		errorObj["error"] = "operation request from unauthorized source is rejected"
+		res.StatusCode = http.StatusInternalServerError
+	} else {
+		//Extract the error and wrap it into the error object
+		data, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			errorObj["error"] = fmt.Sprintf("%s: %s", res.Status, err.Error())
+		} else {
+			if err := json.Unmarshal(data, &errorObj); err != nil {
+				errorObj["error"] = string(data)
+			}
+		}
+	}
+
+	content, err := json.Marshal(errorObj)
+	if err != nil {
+		return err
+	}
+
+	size := len(content)
+	body := ioutil.NopCloser(bytes.NewReader(content))
+	res.Body = body
+	res.ContentLength = int64(size)
+	res.Header.Set(contentLengthHeader, strconv.Itoa(size))
+
 	return nil
 }
 

src/common/const.go

@@ -110,6 +110,9 @@ const (
 	ReloadKey                         = "reload_key"
 	LdapGroupAdminDn                  = "ldap_group_admin_dn"
 	DefaultRegistryControllerEndpoint = "http://registryctl:8080"
+	WithChartMuseum                   = "with_chartmuseum"
+	ChartRepoURL                      = "chart_repository_url"
+	DefaultChartRepoURL               = "http://chartmuseum:9999"
 )
 
 // Shared variable, not allowed to modify

src/ui/api/base.go

@@ -20,6 +20,7 @@ import (
 	"github.com/vmware/harbor/src/common/api"
 	"github.com/vmware/harbor/src/common/security"
 	"github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/ui/config"
 	"github.com/vmware/harbor/src/ui/filter"
 	"github.com/vmware/harbor/src/ui/promgr"
 )
@@ -58,3 +59,20 @@ func (b *BaseController) Prepare() {
 	}
 	b.ProjectMgr = pm
 }
+
+//Init related objects/configurations for the API controllers
+func Init() error {
+	//If chart repository is not enabled then directly return
+	if !config.WithChartMuseum() {
+		return nil
+	}
+
+	chartCtl, err := initializeChartController()
+	if err != nil {
+		return err
+	}
+
+	chartController = chartCtl
+
+	return nil
+}

src/ui/api/chart_repository.go

@@ -2,18 +2,18 @@ package api
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
-	"os"
 	"strings"
 
 	"github.com/vmware/harbor/src/chartserver"
 	hlog "github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/ui/config"
 )
 
 const (
-	backendChartServerAddr  = "BACKEND_CHART_SERVER"
 	namespaceParam          = ":repo"
 	defaultRepo             = "library"
 	rootUploadingEndpoint   = "/api/chartrepo/charts"
@@ -28,7 +28,7 @@ const (
 )
 
 //chartController is a singleton instance
-var chartController = initializeChartController()
+var chartController *chartserver.Controller
 
 //ChartRepositoryAPI provides related API handlers for the chart repository APIs
 type ChartRepositoryAPI struct {
@@ -69,14 +69,6 @@ func (cra *ChartRepositoryAPI) Prepare() {
 	cra.rewriteURLPath(cra.Ctx.Request)
 }
 
-//UploadChartVersionToDefaultNS ...
-func (cra *ChartRepositoryAPI) UploadChartVersionToDefaultNS() {
-	res := make(map[string]interface{})
-	res["result"] = "done"
-	cra.Data["json"] = res
-	cra.ServeJSON()
-}
-
 //GetHealthStatus handles GET /api/chartserver/health
 func (cra *ChartRepositoryAPI) GetHealthStatus() {
 	//Check access
@@ -296,25 +288,25 @@ func (cra *ChartRepositoryAPI) requireAccess(namespace string, accessLevel uint)
 }
 
 //Initialize the chart service controller
-func initializeChartController() *chartserver.Controller {
-	addr := strings.TrimSpace(os.Getenv(backendChartServerAddr))
-	if len(addr) == 0 {
-		hlog.Fatal("The address of chart storage server is not set")
+func initializeChartController() (*chartserver.Controller, error) {
+	addr, err := config.GetChartMuseumEndpoint()
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get the endpoint URL of chart storage server: %s", err.Error())
 	}
 
 	addr = strings.TrimSuffix(addr, "/")
 	url, err := url.Parse(addr)
 	if err != nil {
-		hlog.Fatal("Chart storage server is not correctly configured")
+		return nil, errors.New("Endpoint URL of chart storage server is malformed")
 	}
 
 	controller, err := chartserver.NewController(url)
 	if err != nil {
-		hlog.Fatal("Failed to initialize chart API controller")
+		return nil, errors.New("Failed to initialize chart API controller")
 	}
 
 	hlog.Debugf("Chart storage server is set to %s", url.String())
 	hlog.Info("API controller for chart repository server is successfully initialized")
 
-	return controller
+	return controller, nil
 }

src/ui/api/systeminfo.go

@@ -100,6 +100,7 @@ type GeneralInfo struct {
 	ClairVulnStatus             *models.ClairVulnerabilityStatus `json:"clair_vulnerability_status,omitempty"`
 	RegistryStorageProviderName string                           `json:"registry_storage_provider_name"`
 	ReadOnly                    bool                             `json:"read_only"`
+	WithChartMuseum             bool                             `json:"with_chartmuseum"`
 }
 
 // validate for validating user if an admin.
@@ -179,6 +180,7 @@ func (sia *SystemInfoAPI) GetGeneralInfo() {
 		HarborVersion:               harborVersion,
 		RegistryStorageProviderName: utils.SafeCastString(cfg[common.RegistryStorageProviderName]),
 		ReadOnly:                    config.ReadOnly(),
+		WithChartMuseum:             config.WithChartMuseum(),
 	}
 	if info.WithClair {
 		info.ClairVulnStatus = getClairVulnStatus()

src/ui/config/config.go

@@ -18,6 +18,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -540,3 +541,31 @@ func ReadOnly() bool {
 	}
 	return utils.SafeCastBool(cfg[common.ReadOnly])
 }
+
+// WithChartMuseum returns a bool to indicate if chartmuseum is deployed with Harbor.
+func WithChartMuseum() bool {
+	cfg, err := mg.Get()
+	if err != nil {
+		log.Errorf("Failed to get 'with_chartmuseum' configuration with error: %s; return false as default", err.Error())
+		return false
+	}
+
+	return utils.SafeCastBool(cfg[common.WithChartMuseum])
+}
+
+// GetChartMuseumEndpoint returns the endpoint of the chartmuseum service
+// otherwise an non nil error is returned
+func GetChartMuseumEndpoint() (string, error) {
+	cfg, err := mg.Get()
+	if err != nil {
+		log.Errorf("Failed to get 'chart_repository_url' configuration with error: %s; return false as default", err.Error())
+		return "", err
+	}
+
+	chartEndpoint := strings.TrimSpace(utils.SafeCastString(cfg[common.ChartRepoURL]))
+	if len(chartEndpoint) == 0 {
+		return "", errors.New("empty chartmuseum endpoint")
+	}
+
+	return chartEndpoint, nil
+}

src/ui/main.go

@@ -113,6 +113,11 @@ func main() {
 		log.Error(err)
 	}
 
+	//Init API handler
+	if err := api.Init(); err != nil {
+		log.Fatalf("Failed to initialize API handlers with error: %s", err.Error())
+	}
+
 	//Enable the policy scheduler here.
 	scheduler.DefaultScheduler.Start()
 

src/ui/router.go

@@ -123,21 +123,23 @@ func initRouters() {
 	beego.Router("/registryproxy/*", &controllers.RegistryProxy{}, "*:Handle")
 
 	//APIs for chart repository
-	//Charts are controlled under projects
-	chartRepositoryAPIType := &api.ChartRepositoryAPI{}
-	beego.Router("/api/chartrepo/health", chartRepositoryAPIType, "get:GetHealthStatus")
-	beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "get:ListCharts")
-	beego.Router("/api/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "get:ListChartVersions")
-	beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "get:GetChartVersion")
-	beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "delete:DeleteChartVersion")
-	beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
-	beego.Router("/api/chartrepo/:repo/prov", chartRepositoryAPIType, "post:UploadChartProvFile")
-	beego.Router("/api/chartrepo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
+	if config.WithChartMuseum() {
+		//Charts are controlled under projects
+		chartRepositoryAPIType := &api.ChartRepositoryAPI{}
+		beego.Router("/api/chartrepo/health", chartRepositoryAPIType, "get:GetHealthStatus")
+		beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "get:ListCharts")
+		beego.Router("/api/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "get:ListChartVersions")
+		beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "get:GetChartVersion")
+		beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "delete:DeleteChartVersion")
+		beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
+		beego.Router("/api/chartrepo/:repo/prov", chartRepositoryAPIType, "post:UploadChartProvFile")
+		beego.Router("/api/chartrepo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
 
-	//Repository services
-	beego.Router("/chartrepo/:repo/index.yaml", chartRepositoryAPIType, "get:GetIndexByRepo")
-	beego.Router("/chartrepo/index.yaml", chartRepositoryAPIType, "get:GetIndex")
-	beego.Router("/chartrepo/:repo/charts/:filename", chartRepositoryAPIType, "get:DownloadChart")
+		//Repository services
+		beego.Router("/chartrepo/:repo/index.yaml", chartRepositoryAPIType, "get:GetIndexByRepo")
+		beego.Router("/chartrepo/index.yaml", chartRepositoryAPIType, "get:GetIndex")
+		beego.Router("/chartrepo/:repo/charts/:filename", chartRepositoryAPIType, "get:DownloadChart")
+	}
 
 	//Error pages
 	beego.ErrorController(&controllers.ErrorController{})

tests/resources/Harbor-Util.robot

@@ -41,35 +41,35 @@ Install Harbor to Test Server
     Generate Certificate Authority For Chrome
 
 Up Harbor
-    [Arguments]  ${with_notary}=true  ${with_clair}=true
-    ${rc}  ${output}=  Run And Return Rc And Output  make start -e NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair}
+    [Arguments]  ${with_notary}=true  ${with_clair}=true  ${with_chartmuseum}=true
+    ${rc}  ${output}=  Run And Return Rc And Output  make start -e NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum}
     Log  ${rc}
     Log  ${output}
     Should Be Equal As Integers  ${rc}  0
 
 Down Harbor
-    [Arguments]  ${with_notary}=true  ${with_clair}=true
-    ${rc}  ${output}=  Run And Return Rc And Output  echo "Y" | make down -e NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair}
+    [Arguments]  ${with_notary}=true  ${with_clair}=true  ${with_chartmuseum}=true
+    ${rc}  ${output}=  Run And Return Rc And Output  echo "Y" | make down -e NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum}
     Log  ${rc}
     Log  ${output}
     Should Be Equal As Integers  ${rc}  0
 
 Package Harbor Offline
-    [Arguments]  ${golang_image}=golang:${GOLANG_VERSION}  ${clarity_image}=vmware/harbor-clarity-ui-builder:${CLAIR_BUILDER}  ${with_notary}=true  ${with_clair}=true  ${with_migrator}=true
+    [Arguments]  ${golang_image}=golang:${GOLANG_VERSION}  ${clarity_image}=vmware/harbor-clarity-ui-builder:${CLAIR_BUILDER}  ${with_notary}=true  ${with_clair}=true  ${with_migrator}=true  ${with_chartmuseum}=true
     Log To Console  \nStart Docker Daemon
     Start Docker Daemon Locally
-    Log To Console  \n\nmake package_offline VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} HTTPPROXY=
-    ${rc}  ${output}=  Run And Return Rc And Output  make package_offline VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} HTTPPROXY=
+    Log To Console  \n\nmake package_offline VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} CHARTFLAG=${with_chartmuseum} HTTPPROXY=
+    ${rc}  ${output}=  Run And Return Rc And Output  make package_offline VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} CHARTFLAG=${with_chartmuseum} HTTPPROXY=
     Log  ${rc}
     Log  ${output}
     Should Be Equal As Integers  ${rc}  0
 
 Package Harbor Online
-    [Arguments]  ${golang_image}=golang:${GOLANG_VERSION}  ${clarity_image}=vmware/harbor-clarity-ui-builder:${CLAIR_BUILDER}  ${with_notary}=true  ${with_clair}=true  ${with_migrator}=true
+    [Arguments]  ${golang_image}=golang:${GOLANG_VERSION}  ${clarity_image}=vmware/harbor-clarity-ui-builder:${CLAIR_BUILDER}  ${with_notary}=true  ${with_clair}=true  ${with_migrator}=true  ${with_chartmuseum}=true
     Log To Console  \nStart Docker Daemon
     Start Docker Daemon Locally
-    Log To Console  \nmake package_online VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} HTTPPROXY=
-    ${rc}  ${output}=  Run And Return Rc And Output  make package_online VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} HTTPPROXY=
+    Log To Console  \nmake package_online VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} CHARTFLAG=${with_chartmuseum} HTTPPROXY=
+    ${rc}  ${output}=  Run And Return Rc And Output  make package_online VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} CHARTFLAG=${with_chartmuseum} HTTPPROXY=
     Log  ${rc}
     Log  ${output}
     Should Be Equal As Integers  ${rc}  0
@@ -116,8 +116,8 @@ Enable Notary Client
     Log  ${output}
 
 Prepare
-    [Arguments]  ${with_notary}=true  ${with_clair}=true
-    ${rc}  ${output}=  Run And Return Rc And Output  make prepare -e NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair}
+    [Arguments]  ${with_notary}=true  ${with_clair}=true  ${with_chartmuseum}=true
+    ${rc}  ${output}=  Run And Return Rc And Output  make prepare -e NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum}
     Log  ${rc}
     Log  ${output}
     Should Be Equal As Integers  ${rc}  0
@@ -150,14 +150,14 @@ Prepare Cert
     Should Be Equal As Integers  ${rc}  0
 
 Compile and Up Harbor With Source Code
-    [Arguments]  ${golang_image}=golang:${GOLANG_VERSION}  ${clarity_image}=vmware/harbor-clarity-ui-builder:${CLAIR_BUILDER}  ${with_notary}=true  ${with_clair}=true
+    [Arguments]  ${golang_image}=golang:${GOLANG_VERSION}  ${clarity_image}=vmware/harbor-clarity-ui-builder:${CLAIR_BUILDER}  ${with_notary}=true  ${with_clair}=true  ${with_chartmuseum}=true
     ${rc}  ${output}=  Run And Return Rc And Output  docker pull ${clarity_image}
     Log  ${output}
     Should Be Equal As Integers  ${rc}  0
     ${rc}  ${output}=  Run And Return Rc And Output  docker pull ${golang_image}
     Log  ${output}
     Should Be Equal As Integers  ${rc}  0
-    ${rc}  ${output}=  Run And Return Rc And Output  make install GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} HTTPPROXY=
+    ${rc}  ${output}=  Run And Return Rc And Output  make install GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum} HTTPPROXY=
     Log  ${output}
     Should Be Equal As Integers  ${rc}  0
     Sleep  20