diff --git a/Makefile b/Makefile index c4afa8ba2..64395c0f0 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,8 @@ REBUILDCLARITYFLAG=false NEWCLARITYVERSION= BUILDBIN=false MIGRATORFLAG=false +# enable/disable chart repo supporting +CHARTFLAG=false # version prepare # for docker image tag @@ -104,6 +106,8 @@ CLAIRVERSION=v2.0.4 CLAIRDBVERSION=$(VERSIONTAG) MIGRATORVERSION=$(VERSIONTAG) REDISVERSION=$(VERSIONTAG) +# version of chartmuseum +CHARTMUSEUMVERSION=v0.7.1 #clarity parameters CLARITYIMAGE=vmware/harbor-clarity-ui-builder[:tag] @@ -129,7 +133,7 @@ GOINSTALL=$(GOCMD) install GOTEST=$(GOCMD) test GODEP=$(GOTEST) -i GOFMT=gofmt -w -GOBUILDIMAGE=reg.mydomain.com/library/harborgo[:tag] +GOBUILDIMAGE=golang:1.9.2 GOBUILDPATH=$(GOBASEPATH)/harbor GOIMAGEBUILDCMD=/usr/local/go/bin/go GOIMAGEBUILD=$(GOIMAGEBUILDCMD) build @@ -168,6 +172,10 @@ endif ifeq ($(CLAIRFLAG), true) PREPARECMD_PARA+= --with-clair endif +# append chartmuseum parameters if set +ifeq ($(CHARTFLAG), true) + PREPARECMD_PARA+= --with-chartmuseum +endif # makefile MAKEFILEPATH_PHOTON=$(MAKEPATH)/photon @@ -183,6 +191,7 @@ DOCKERIMAGENAME_JOBSERVICE=vmware/harbor-jobservice DOCKERIMAGENAME_LOG=vmware/harbor-log DOCKERIMAGENAME_DB=vmware/harbor-db DOCKERIMAGENAME_CLARITY=vmware/harbor-clarity-ui-builder +DOCKERIMAGENAME_CHART_SERVER=vmware/chartmuseum-photon DOCKERIMAGENAME_REGCTL=vmware/harbor-registryctl # docker-compose files @@ -193,6 +202,8 @@ DOCKERCOMPOSENOTARYTPLFILENAME=docker-compose.notary.tpl DOCKERCOMPOSENOTARYFILENAME=docker-compose.notary.yml DOCKERCOMPOSECLAIRTPLFILENAME=docker-compose.clair.tpl DOCKERCOMPOSECLAIRFILENAME=docker-compose.clair.yml +DOCKERCOMPOSECHARTMUSEUMTPLFILENAME=docker-compose.chartmuseum.tpl +DOCKERCOMPOSECHARTMUSEUMFILENAME=docker-compose.chartmuseum.yml SEDCMD=$(shell which sed) @@ -247,6 +258,13 @@ endif ifeq ($(MIGRATORFLAG), true) DOCKERSAVE_PARA+= vmware/harbor-migrator:$(MIGRATORVERSION) endif +# append chartmuseum parameters if set +ifeq ($(CHARTFLAG), true) + DOCKERSAVE_PARA+= $(DOCKERIMAGENAME_CHART_SERVER):$(CHARTMUSEUMVERSION)-$(VERSIONTAG) + PACKAGE_OFFLINE_PARA+= $(HARBORPKG)/$(DOCKERCOMPOSECHARTMUSEUMFILENAME) + PACKAGE_ONLINE_PARA+= $(HARBORPKG)/$(DOCKERCOMPOSECHARTMUSEUMFILENAME) + DOCKERCOMPOSE_LIST+= -f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECHARTMUSEUMFILENAME) +endif version: @printf $(UIVERSIONTAG) > $(VERSIONFILEPATH)/$(VERSIONFILENAME); @@ -294,9 +312,10 @@ build: make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) -e MARIADBVERSION=$(MARIADBVERSION) \ -e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) \ -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \ - -e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) + -e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \ + -e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER) -modify_composefile: modify_composefile_notary modify_composefile_clair +modify_composefile: modify_composefile_notary modify_composefile_clair modify_composefile_chartmuseum @echo "preparing docker-compose file..." @cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSETPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME) @cp $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSETPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSEFILENAME) @@ -323,6 +342,11 @@ modify_composefile_clair: @cp $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSECLAIRTPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSECLAIRFILENAME) @$(SEDCMD) -i 's/__clair_version__/$(CLAIRVERSION)-$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSECLAIRFILENAME) +modify_composefile_chartmuseum: + @echo "preparing docker-compose chartmuseum file..." + @cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECHARTMUSEUMTPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECHARTMUSEUMFILENAME) + @$(SEDCMD) -i 's/__chartmuseum_version__/$(CHARTMUSEUMVERSION)-$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECHARTMUSEUMFILENAME) + modify_sourcefiles: @echo "change mode of source files." @chmod 600 $(MAKEPATH)/common/templates/notary/notary-signer.key @@ -421,7 +445,7 @@ start: @echo "Start complete. You can visit harbor now." down: - @echo "Please make sure to set -e NOTARYFLAG=true/CLAIRFLAG=true if you are using Notary/CLAIR in Harbor, otherwise the Notary/CLAIR containers cannot be stop automaticlly." + @echo "Please make sure to set -e NOTARYFLAG=true/CLAIRFLAG=true/CHARTFLAG=true if you are using Notary/CLAIR/Chartmuseum in Harbor, otherwise the Notary/CLAIR/Chartmuseum containers cannot be stop automaticlly." @while [ -z "$$CONTINUE" ]; do \ read -r -p "Type anything but Y or y to exit. [Y/N]: " CONTINUE; \ done ; \ diff --git a/make/common/templates/adminserver/env b/make/common/templates/adminserver/env index 9fbeafb31..fa34d342b 100644 --- a/make/common/templates/adminserver/env +++ b/make/common/templates/adminserver/env @@ -61,5 +61,7 @@ REGISTRY_STORAGE_PROVIDER_NAME=$storage_provider_name READ_ONLY=false SKIP_RELOAD_ENV_PATTERN=$skip_reload_env_pattern RELOAD_KEY=$reload_key +CHART_REPOSITORY_URL=$chart_repository_url LDAP_GROUP_ADMIN_DN=$ldap_group_admin_dn -REGISTRY_CONTROLLER_URL=$registry_controller_url \ No newline at end of file +REGISTRY_CONTROLLER_URL=$registry_controller_url +WITH_CHARTMUSEUM=$with_chartmuseum diff --git a/make/common/templates/chartserver/env b/make/common/templates/chartserver/env new file mode 100644 index 000000000..fca2ab0e3 --- /dev/null +++ b/make/common/templates/chartserver/env @@ -0,0 +1,41 @@ +## Settings should be set +PORT=9999 + +# Only support redis now. If redis is setup, then enable cache +CACHE=$cache_store +CACHE_REDIS_ADDR=$cache_redis_addr +CACHE_REDIS_PASSWORD=$cache_redis_password +CACHE_REDIS_DB=$cache_redis_db_index + +# Credential for internal communication +BASIC_AUTH_USER=chart_controller +BASIC_AUTH_PASS=$ui_secret + +# Multiple tenants +# Must be set with 1 to support project namespace +DEPTH=1 + +# Backend storage driver: e.g. "local", "amazon", "google" etc. +STORAGE=$storage_driver + +# Storage driver settings +$all_storage_driver_configs + +## Settings with default values. Just put here for future changes +DEBUG=false +LOG_JSON=true +DISABLE_METRICS=false +DISABLE_API=false +DISABLE_STATEFILES=false +ALLOW_OVERWRITE=false +CHART_URL= +AUTH_ANONYMOUS_GET=false +TLS_CERT= +TLS_KEY= +CONTEXT_PATH= +INDEX_LIMIT=0 +MAX_STORAGE_OBJECTS=0 +MAX_UPLOAD_SIZE=20971520 +CHART_POST_FORM_FIELD_NAME=chart +PROV_POST_FORM_FIELD_NAME=prov + diff --git a/make/common/templates/ui/env b/make/common/templates/ui/env index ad1a9fd69..7ec4ad559 100644 --- a/make/common/templates/ui/env +++ b/make/common/templates/ui/env @@ -7,3 +7,4 @@ ADMINSERVER_URL=$adminserver_url UAA_CA_ROOT=/etc/ui/certificates/uaa_ca.pem _REDIS_URL=$redis_url SYNC_REGISTRY=false +CHART_CACHE_DRIVER=$chart_cache_driver diff --git a/make/docker-compose.chartmuseum.tpl b/make/docker-compose.chartmuseum.tpl new file mode 100644 index 000000000..cdd7f856d --- /dev/null +++ b/make/docker-compose.chartmuseum.tpl @@ -0,0 +1,32 @@ +version: '2' +services: + ui: + networks: + harbor-chartmuseum: + aliases: + - harbor-ui + redis: + networks: + harbor-chartmuseum: + aliases: + - redis + chartmuseum: + container_name: chartmuseum + image: vmware/chartmuseum-photon:__chartmuseum_version__ + restart: always + networks: + - harbor-chartmuseum + depends_on: + - redis + volumes: + - /data/chart_storage:/chart_storage:z + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + tag: "chartmuseum" + env_file: + ./common/config/chartserver/env +networks: + harbor-chartmuseum: + external: false diff --git a/make/install.sh b/make/install.sh index 069c4b722..27225f160 100755 --- a/make/install.sh +++ b/make/install.sh @@ -60,6 +60,9 @@ with_notary=$false with_clair=$false # HA mode is not enabled by default harbor_ha=$false +# chartmuseum is not enabled by default +with_chartmuseum=$false + while [ $# -gt 0 ]; do case $1 in --help) @@ -71,6 +74,8 @@ while [ $# -gt 0 ]; do with_clair=true;; --ha) harbor_ha=true;; + --with-chartmuseum) + with_chartmuseum=true;; *) note "$usage" exit 1;; @@ -173,6 +178,11 @@ if [ $harbor_ha ] then prepare_para="${prepare_para} --ha" fi +if [ $with_chartmuseum ] +then + prepare_para="${prepare_para} --with-chartmuseum" +fi + ./prepare $prepare_para echo "" @@ -186,6 +196,10 @@ if [ $with_clair ] then docker_compose_list="${docker_compose_list} -f docker-compose.clair.yml" fi +if [ $with_chartmuseum ] +then + docker_compose_list="${docker_compose_list} -f docker-compose.chartmuseum.yml" +fi if [ -n "$(docker-compose $docker_compose_list ps -q)" ] then diff --git a/make/photon/Makefile b/make/photon/Makefile index ec69b6d0a..3a409155c 100644 --- a/make/photon/Makefile +++ b/make/photon/Makefile @@ -89,6 +89,13 @@ DOCKERFILEPATH_MIGRATOR=$(TOOLSPATH)/migration DOCKERFILENAME_MIGRATOR=Dockerfile DOCKERIMAGENAME_MIGRATOR=vmware/harbor-migrator +# for chart server (chartmuseum) +DOCKERFILEPATH_CHART_SERVER=$(DOCKERFILEPATH)/chartserver +DOCKERFILENAME_CHART_SERVER=Dockerfile +CHART_SERVER_CODE_BASE=github.com/helm/chartmuseum +CHART_SERVER_MAIN_PATH=cmd/chartmuseum +CHART_SERVER_BIN_NAME=chartm + _build_db: @echo "building db container for photon..." @cd $(DOCKERFILEPATH_DB) && $(DOCKERBUILD) -f $(DOCKERFILEPATH_DB)/$(DOCKERFILENAME_DB) -t $(DOCKERIMAGENAME_DB):$(VERSIONTAG) . @@ -127,6 +134,20 @@ _build_clair: rm -rf $(DOCKERFILEPATH_CLAIR)/binary; \ echo "Done." ; \ fi + +_build_chart_server: + @if [ "$(CHARTFLAG)" = "true" ] ; then \ + if [ "$(BUILDBIN)" != "true" ] ; then \ + rm -rf $(DOCKERFILEPATH_CHART_SERVER)/binary && mkdir -p $(DOCKERFILEPATH_CHART_SERVER)/binary && \ + $(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/chartm, $(DOCKERFILEPATH_CHART_SERVER)/binary/chartm); \ + else \ + cd $(DOCKERFILEPATH_CHART_SERVER) && $(DOCKERFILEPATH_CHART_SERVER)/builder $(GOBUILDIMAGE) $(CHART_SERVER_CODE_BASE) $(CHARTMUSEUMVERSION) $(CHART_SERVER_MAIN_PATH) $(CHART_SERVER_BIN_NAME); \ + fi ; \ + echo "building chartmuseum container for photon..." ; \ + cd $(DOCKERFILEPATH_CHART_SERVER) && $(DOCKERBUILD) -f $(DOCKERFILEPATH_CHART_SERVER)/$(DOCKERFILENAME_CHART_SERVER) -t $(DOCKERIMAGENAME_CHART_SERVER):$(CHARTMUSEUMVERSION)-$(VERSIONTAG) . ; \ + rm -rf $(DOCKERFILEPATH_CHART_SERVER)/binary; \ + echo "Done." ; \ + fi _build_nginx: @echo "building nginx container for photon..." @@ -182,7 +203,7 @@ define _get_binary $(WGET) --timeout 30 --no-check-certificate $1 -O $2 endef -build: _build_db _build_adminiserver _build_ui _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_redis _build_migrator +build: _build_db _build_adminiserver _build_ui _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_redis _build_migrator _build_chart_server cleanimage: @echo "cleaning image for photon..." diff --git a/make/photon/chartserver/Dockerfile b/make/photon/chartserver/Dockerfile new file mode 100644 index 000000000..f06c03609 --- /dev/null +++ b/make/photon/chartserver/Dockerfile @@ -0,0 +1,22 @@ +FROM photon:1.0 + +RUN tdnf distro-sync -y \ + && tdnf erase vim -y \ + && tdnf install -y git shadow sudo bzr rpm xz python-xml >>/dev/null\ + && tdnf clean all \ + && mkdir /chartserver/ \ + && groupadd -r -g 10000 chartuser \ + && useradd --no-log-init -m -r -g 10000 -u 10000 chartuser +COPY ./binary/chartm /chartserver/ +COPY docker-entrypoint.sh /docker-entrypoint.sh + +EXPOSE 9999 + +RUN chown -R 10000:10000 /chartserver \ + && chmod u+x /chartserver/chartm \ + && chmod u+x /docker-entrypoint.sh + + +HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS 127.0.0.1:9999/health || exit 1 + +ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/make/photon/chartserver/builder b/make/photon/chartserver/builder new file mode 100755 index 000000000..579eadf6b --- /dev/null +++ b/make/photon/chartserver/builder @@ -0,0 +1,33 @@ +#!/bin/bash + +set +e + +usage(){ + echo "Usage: builder " + echo "e.g: builder golang:1.9.2 github.com/helm/chartmuseum v0.7.1 cmd/chartmuseum chartm" + exit 1 +} + +if [ $# != 5 ]; then + usage +fi + +GOLANG_IMAGE="$1" +CODE_PATH="$2" +CODE_VERSION="$3" +MAIN_GO_PATH="$4" +BIN_NAME="$5" + +set -e + +cd `dirname $0` +cur=$PWD + +mkdir -p binary +rm -rf binary/$BIN_NAME || true +cp compile.sh binary/ + +docker run -it -v $cur/binary:/go/bin --name golang_code_builder $GOLANG_IMAGE /bin/bash /go/bin/compile.sh $CODE_PATH $CODE_VERSION $MAIN_GO_PATH $BIN_NAME + +#Clear +docker rm -f golang_code_builder diff --git a/make/photon/chartserver/compile.sh b/make/photon/chartserver/compile.sh new file mode 100644 index 000000000..dca0d6c1d --- /dev/null +++ b/make/photon/chartserver/compile.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set +e + +usage(){ + echo "Usage: compile.sh " + echo "e.g: compile.sh github.com/helm/chartmuseum v0.5.1 cmd/chartmuseum chartm" + exit 1 +} + +if [ $# != 4 ]; then + usage +fi + +CODE_PATH="$1" +VERSION="$2" +MAIN_GO_PATH="$3" +BIN_NAME="$4" + +#Get the source code of chartmusem +go get $CODE_PATH + +set -e + +#Checkout the released tag branch +cd /go/src/$CODE_PATH +git checkout tags/$VERSION -b $VERSION + +#Install the go dep tool to restore the package dependencies +curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh +dep ensure + +#Compile +cd /go/src/$CODE_PATH/$MAIN_GO_PATH && go build -a -o $BIN_NAME +mv $BIN_NAME /go/bin/ diff --git a/make/photon/chartserver/docker-entrypoint.sh b/make/photon/chartserver/docker-entrypoint.sh new file mode 100644 index 000000000..0d67c2a0f --- /dev/null +++ b/make/photon/chartserver/docker-entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +#/chart_storage is the directory in the contaienr for storing the chart artifacts +#if storage driver is set to 'local' +if [ -d /chart_storage ]; then + chown 10000:10000 -R /chart_storage +fi + +# Start the server process +sudo -E -H -u \#10000 sh -c "/chartserver/chartm" #Parameters are set by ENV +set +e diff --git a/make/prepare b/make/prepare index c1ce16a53..86cbe1e22 100755 --- a/make/prepare +++ b/make/prepare @@ -69,6 +69,16 @@ def validate(conf, args): if project_creation != "everyone" and project_creation != "adminonly": raise Exception("Error invalid value for project_creation_restriction: %s" % project_creation) + + valid_storage_drivers = ["filesystem", "azure", "gcs", "s3", "swift", "oss"] + storage_provider_name = rcp.get("configuration", "registry_storage_provider_name").strip() + if storage_provider_name not in valid_storage_drivers: + raise Exception("Error: storage driver %s is not supported, only the following ones are supported: %s" % (storage_provider_name, ",".join(valid_storage_drivers))) + + storage_provider_config = rcp.get("configuration", "registry_storage_provider_config").strip() + if storage_provider_name != "filesystem": + if storage_provider_config == "": + raise Exception("Error: no provider configurations are provided for provider %s" % storage_provider_name) #To meet security requirement #By default it will change file mode to 0600, and make the owner of the file to 10000:10000 @@ -188,6 +198,7 @@ parser.add_argument('--with-notary', dest='notary_mode', default=False, action=' parser.add_argument('--with-clair', dest='clair_mode', default=False, action='store_true', help="the Harbor instance is to be deployed with clair") parser.add_argument('--ha', dest='ha_mode', default=False, action='store_true', help="the Harbor instance is to be deployed in HA mode") parser.add_argument('--yes', dest='yes', default=False, action='store_true', help="Answer yes to all questions") +parser.add_argument('--with-chartmuseum', dest='chart_mode', default=False, action='store_true', help="the Harbor instance is to be deployed with chart repository supporting") args = parser.parse_args() delfile(config_dir) @@ -322,6 +333,7 @@ token_service_url = "http://ui:8080/service/token" jobservice_url = "http://jobservice:8080" clair_url = "http://clair:6060" notary_url = "http://notary-server:4443" +chart_repository_url = "http://chartmuseum:9999" if len(admiral_url) != 0 and admiral_url != "NA": #VIC overwrites the data volume path, which by default should be same as the value of secretkey_path @@ -409,15 +421,24 @@ render(os.path.join(templates_dir, "adminserver", "env"), notary_url=notary_url, reload_key=reload_key, skip_reload_env_pattern=skip_reload_env_pattern, - registry_controller_url = registry_controller_url + chart_repository_url=chart_repository_url, + registry_controller_url = registry_controller_url, + with_chartmuseum=args.chart_mode ) +# set cache for chart repo server +# default set 'memory' mode, if redis is configured then set to 'redis' +chart_cache_driver = "memory" +if len(redis_url) > 0: + chart_cache_driver = "redis" + render(os.path.join(templates_dir, "ui", "env"), ui_conf_env, ui_secret=ui_secret, jobservice_secret=jobservice_secret, redis_url = redis_url, - adminserver_url = adminserver_url + adminserver_url = adminserver_url, + chart_cache_driver = chart_cache_driver ) registry_config_file_ha = "config_ha.yml" @@ -635,5 +656,99 @@ if args.clair_mode: if args.ha_mode: prepare_ha(rcp, args) +# config chart repository +if args.chart_mode: + chartm_temp_dir = os.path.join(templates_dir, "chartserver") + chartm_config_dir = os.path.join(config_dir, "chartserver") + chartm_env = os.path.join(config_dir, "chartserver", "env") + + if not os.path.isdir(chartm_config_dir): + print ("Create config folder: %s" % chartm_config_dir) + os.makedirs(chartm_config_dir) + + # process redis info + cache_store = "" + cache_redis_password = "" + cache_redis_addr = "" + cache_redis_db_index = 0 + if redis_url and redis_url.strip(): + cache_store = "redis" + segments = redis_url.split(',', 3) + for index, r_cfg in enumerate(segments): + # the addr:port + if index == 0: + cache_redis_addr = r_cfg + # the password if existing + elif index == 2: + cache_redis_password = r_cfg + # the database index if existing + elif index == 3: + cache_redis_db_index = r_cfg + + # process storage info + #default using local file system + storage_driver = "local" + # storage provider configurations + # please be aware that, we do not check the validations of the values for the specified keys + # convert the configs to config map + storage_provider_configs = storage_provider_config.split(",") + storgae_provider_confg_map = {} + storage_provider_config_options = [] + + for k_v in storage_provider_configs: + if len(k_v) > 0: + kvs = k_v.split(": ") # add space suffix to avoid existing ":" in the value + if len(kvs) == 2: + #key must not be empty + if kvs[0].strip() != "": + storgae_provider_confg_map[kvs[0].strip()] = kvs[1].strip() + + if storage_provider_name == "s3": + # aws s3 storage + storage_driver = "amazon" + storage_provider_config_options.append("STORAGE_AMAZON_BUCKET=%s" % storgae_provider_confg_map.get("bucket", "")) + storage_provider_config_options.append("STORAGE_AMAZON_PREFIX=%s" % storgae_provider_confg_map.get("rootdirectory", "")) + storage_provider_config_options.append("STORAGE_AMAZON_REGION=%s" % storgae_provider_confg_map.get("region", "")) + storage_provider_config_options.append("STORAGE_AMAZON_ENDPOINT=%s" % storgae_provider_confg_map.get("regionendpoint", "")) + elif storage_provider_name == "gcs": + # google cloud storage + storage_driver = "google" + storage_provider_config_options.append("STORAGE_GOOGLE_BUCKET=%s" % storgae_provider_confg_map.get("bucket", "")) + storage_provider_config_options.append("STORAGE_GOOGLE_PREFIX=%s" % storgae_provider_confg_map.get("rootdirectory", "")) + elif storage_provider_name == "azure": + # azure storage + storage_driver = "microsoft" + storage_provider_config_options.append("STORAGE_MICROSOFT_CONTAINER=%s" % storgae_provider_confg_map.get("container", "")) + storage_provider_config_options.append("STORAGE_MICROSOFT_PREFIX=/azure/harbor/charts") + elif storage_provider_name == "swift": + # open stack swift + storage_driver = "openstack" + storage_provider_config_options.append("STORAGE_OPENSTACK_CONTAINER=%s" % storgae_provider_confg_map.get("container", "")) + storage_provider_config_options.append("STORAGE_OPENSTACK_PREFIX=%s" % storgae_provider_confg_map.get("rootdirectory", "")) + storage_provider_config_options.append("STORAGE_OPENSTACK_REGION=%s" % storgae_provider_confg_map.get("region", "")) + elif storage_provider_name == "oss": + # aliyun OSS + storage_driver = "alibaba" + storage_provider_config_options.append("STORAGE_ALIBABA_BUCKET=%s" % storgae_provider_confg_map.get("bucket", "")) + storage_provider_config_options.append("STORAGE_ALIBABA_PREFIX=%s" % storgae_provider_confg_map.get("rootdirectory", "")) + storage_provider_config_options.append("STORAGE_ALIBABA_ENDPOINT=%s" % storgae_provider_confg_map.get("endpoint", "")) + else: + # use local file system + storage_provider_config_options.append("STORAGE_LOCAL_ROOTDIR=/chart_storage") + + # generate storage provider configuration + all_storage_provider_configs = ('\n').join(storage_provider_config_options) + + render(os.path.join(chartm_temp_dir, "env"), + chartm_env, + cache_store=cache_store, + cache_redis_addr=cache_redis_addr, + cache_redis_password=cache_redis_password, + cache_redis_db_index=cache_redis_db_index, + ui_secret=ui_secret, + storage_driver=storage_driver, + all_storage_driver_configs=all_storage_provider_configs) + + FNULL.close() print("The configuration files are ready, please use docker-compose to start the service.") diff --git a/src/adminserver/systemcfg/store/database/driver_db.go b/src/adminserver/systemcfg/store/database/driver_db.go index 42e51635d..d434bef16 100644 --- a/src/adminserver/systemcfg/store/database/driver_db.go +++ b/src/adminserver/systemcfg/store/database/driver_db.go @@ -49,6 +49,7 @@ var ( common.LDAPVerifyCert: true, common.UAAVerifyCert: true, common.ReadOnly: true, + common.WithChartMuseum: true, } mapKeys = map[string]bool{ common.ScanAllPolicy: true, diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index 61dda2ffc..07285cb03 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -166,6 +166,11 @@ var ( }, common.ReloadKey: "RELOAD_KEY", common.LdapGroupAdminDn: "LDAP_GROUP_ADMIN_DN", + common.ChartRepoURL: "CHART_REPOSITORY_URL", + common.WithChartMuseum: &parser{ + env: "WITH_CHARTMUSEUM", + parse: parseStringToBool, + }, } // configurations need read from environment variables @@ -220,6 +225,11 @@ var ( common.ClairURL: "CLAIR_URL", common.NotaryURL: "NOTARY_URL", common.DatabaseType: "DATABASE_TYPE", + common.ChartRepoURL: "CHART_REPOSITORY_URL", + common.WithChartMuseum: &parser{ + env: "WITH_CHARTMUSEUM", + parse: parseStringToBool, + }, } ) diff --git a/src/chartserver/client.go b/src/chartserver/client.go index 0deda03e9..41c0ffb88 100644 --- a/src/chartserver/client.go +++ b/src/chartserver/client.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "strings" "time" ) @@ -42,12 +43,17 @@ func NewChartClient(credentail *Credential) *ChartClient { //Create http client } //GetContent get the bytes from the specified url -func (cc *ChartClient) GetContent(url string) ([]byte, error) { - if len(strings.TrimSpace(url)) == 0 { +func (cc *ChartClient) GetContent(addr string) ([]byte, error) { + if len(strings.TrimSpace(addr)) == 0 { return nil, errors.New("empty url is not allowed") } - request, err := http.NewRequest(http.MethodGet, url, nil) + fullURI, err := url.Parse(addr) + if err != nil { + return nil, fmt.Errorf("invalid url: %s", err.Error()) + } + + request, err := http.NewRequest(http.MethodGet, addr, nil) if err != nil { return nil, err } @@ -69,7 +75,7 @@ func (cc *ChartClient) GetContent(url string) ([]byte, error) { defer response.Body.Close() if response.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to retrieve content from url '%s' with error: %s", url, content) + return nil, fmt.Errorf("failed to retrieve content from '%s' with error: %s", fullURI.Path, content) } return content, nil diff --git a/src/chartserver/controller_test.go b/src/chartserver/controller_test.go index 7e1fd7519..be86f45a7 100644 --- a/src/chartserver/controller_test.go +++ b/src/chartserver/controller_test.go @@ -179,7 +179,7 @@ func TestResponseRewrite(t *testing.T) { if msg, ok := errObj["error"]; !ok { t.Fatal("Expect an error message from server but got nothing") } else { - if !strings.Contains(msg.(string), "operation request from unauthentic source is rejected") { + if !strings.Contains(msg.(string), "operation request from unauthorized source is rejected") { t.Fatal("Missing the required error message") } } diff --git a/src/chartserver/manipulation_handler.go b/src/chartserver/manipulation_handler.go index e246d4f6c..ae79dafbd 100644 --- a/src/chartserver/manipulation_handler.go +++ b/src/chartserver/manipulation_handler.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/url" + "path" "strings" "github.com/ghodss/yaml" @@ -43,10 +44,10 @@ type ManipulationHandler struct { //ListCharts lists all the charts under the specified namespace func (mh *ManipulationHandler) ListCharts(w http.ResponseWriter, req *http.Request) { - rootURL := strings.TrimSuffix(mh.backendServerAddress.String(), "/") - fullURL := fmt.Sprintf("%s%s", rootURL, req.RequestURI) + url := strings.TrimPrefix(req.URL.String(), "/") + url = fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), url) - content, err := mh.apiClient.GetContent(fullURL) + content, err := mh.apiClient.GetContent(url) if err != nil { writeInternalError(w, err) return @@ -76,7 +77,7 @@ func (mh *ManipulationHandler) GetChart(w http.ResponseWriter, req *http.Request //This handler should return the details of the chart version, //maybe including metadata,dependencies and values etc. func (mh *ManipulationHandler) GetChartVersion(w http.ResponseWriter, req *http.Request) { - chartV, err := mh.getChartVersion(req.RequestURI) + chartV, err := mh.getChartVersion(req.URL.String()) if err != nil { writeInternalError(w, err) return @@ -149,11 +150,10 @@ func (mh *ManipulationHandler) DeleteChartVersion(w http.ResponseWriter, req *ht } //Get the basic metadata of chart version -func (mh *ManipulationHandler) getChartVersion(path string) (*helm_repo.ChartVersion, error) { - rootURL := strings.TrimSuffix(mh.backendServerAddress.String(), "/") - fullURL := fmt.Sprintf("%s%s", rootURL, path) +func (mh *ManipulationHandler) getChartVersion(subPath string) (*helm_repo.ChartVersion, error) { + url := fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), strings.TrimPrefix(subPath, "/")) - content, err := mh.apiClient.GetContent(fullURL) + content, err := mh.apiClient.GetContent(url) if err != nil { return nil, err } @@ -167,9 +167,9 @@ func (mh *ManipulationHandler) getChartVersion(path string) (*helm_repo.ChartVer } //Get the content bytes of the chart version -func (mh *ManipulationHandler) getChartVersionContent(namespace string, path string) ([]byte, error) { - rootURL := strings.TrimSuffix(mh.backendServerAddress.String(), "/") - fullPath := fmt.Sprintf("%s/%s/%s", rootURL, namespace, path) +func (mh *ManipulationHandler) getChartVersionContent(namespace string, subPath string) ([]byte, error) { + url := path.Join(namespace, subPath) + url = fmt.Sprintf("%s/%s", mh.backendServerAddress.String(), url) - return mh.apiClient.GetContent(fullPath) + return mh.apiClient.GetContent(url) } diff --git a/src/chartserver/repo_handler.go b/src/chartserver/repo_handler.go index 401cdaad1..6f510dd12 100644 --- a/src/chartserver/repo_handler.go +++ b/src/chartserver/repo_handler.go @@ -6,7 +6,7 @@ import ( "fmt" "net/http" "net/url" - "strings" + "path" "sync" "sync/atomic" "time" @@ -14,6 +14,8 @@ import ( "github.com/ghodss/yaml" "github.com/vmware/harbor/src/ui/filter" helm_repo "k8s.io/helm/pkg/repo" + + hlog "github.com/vmware/harbor/src/common/utils/log" ) const ( @@ -209,8 +211,9 @@ func (rh *RepositoryHandler) DownloadChartObject(w http.ResponseWriter, req *htt //Get the index yaml file under the specified namespace from the backend server func (rh *RepositoryHandler) getIndexYamlWithNS(namespace string) (*helm_repo.IndexFile, error) { //Join url path - rootURL := strings.TrimSuffix(rh.backendServerAddress.String(), "/") - url := fmt.Sprintf("%s/%s/index.yaml", rootURL, namespace) + url := path.Join(namespace, "index.yaml") + url = fmt.Sprintf("%s/%s", rh.backendServerAddress.String(), url) + hlog.Debugf("Getting index.yaml from '%s'", url) content, err := rh.apiClient.GetContent(url) if err != nil { @@ -238,7 +241,7 @@ func (rh *RepositoryHandler) mergeIndexFile(namespace string, version.Name = nameWithNS //Currently there is only one url for index, url := range version.URLs { - version.URLs[index] = fmt.Sprintf("%s/%s", namespace, url) + version.URLs[index] = path.Join(namespace, url) } } diff --git a/src/chartserver/reverse_proxy.go b/src/chartserver/reverse_proxy.go index 4ee0bc290..d97e2ff5f 100644 --- a/src/chartserver/reverse_proxy.go +++ b/src/chartserver/reverse_proxy.go @@ -3,6 +3,7 @@ package chartserver import ( "bytes" "encoding/json" + "fmt" "io/ioutil" "log" "net/http" @@ -73,25 +74,41 @@ func director(target *url.URL, cred *Credential, req *http.Request) { //Modify the http response func modifyResponse(res *http.Response) error { - //Detect the 401 code, if it is, - //overwrite it to 500. - //We also re-write the error content - if res.StatusCode == http.StatusUnauthorized { - errorObj := make(map[string]string) - errorObj["error"] = "operation request from unauthentic source is rejected" - content, err := json.Marshal(errorObj) - if err != nil { - return err - } - - size := len(content) - body := ioutil.NopCloser(bytes.NewReader(content)) - res.Body = body - res.ContentLength = int64(size) - res.Header.Set(contentLengthHeader, strconv.Itoa(size)) - res.StatusCode = http.StatusInternalServerError + //Accept cases + //Success or redirect + if res.StatusCode >= http.StatusOK && res.StatusCode <= http.StatusTemporaryRedirect { + return nil } + //Detect the 401 code, if it is,overwrite it to 500. + //We also re-write the error content to structural error object + errorObj := make(map[string]string) + if res.StatusCode == http.StatusUnauthorized { + errorObj["error"] = "operation request from unauthorized source is rejected" + res.StatusCode = http.StatusInternalServerError + } else { + //Extract the error and wrap it into the error object + data, err := ioutil.ReadAll(res.Body) + if err != nil { + errorObj["error"] = fmt.Sprintf("%s: %s", res.Status, err.Error()) + } else { + if err := json.Unmarshal(data, &errorObj); err != nil { + errorObj["error"] = string(data) + } + } + } + + content, err := json.Marshal(errorObj) + if err != nil { + return err + } + + size := len(content) + body := ioutil.NopCloser(bytes.NewReader(content)) + res.Body = body + res.ContentLength = int64(size) + res.Header.Set(contentLengthHeader, strconv.Itoa(size)) + return nil } diff --git a/src/common/const.go b/src/common/const.go index e83629020..b86900a8b 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -110,6 +110,9 @@ const ( ReloadKey = "reload_key" LdapGroupAdminDn = "ldap_group_admin_dn" DefaultRegistryControllerEndpoint = "http://registryctl:8080" + WithChartMuseum = "with_chartmuseum" + ChartRepoURL = "chart_repository_url" + DefaultChartRepoURL = "http://chartmuseum:9999" ) // Shared variable, not allowed to modify diff --git a/src/ui/api/base.go b/src/ui/api/base.go index 4b40b3563..a2eb45554 100644 --- a/src/ui/api/base.go +++ b/src/ui/api/base.go @@ -20,6 +20,7 @@ import ( "github.com/vmware/harbor/src/common/api" "github.com/vmware/harbor/src/common/security" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/config" "github.com/vmware/harbor/src/ui/filter" "github.com/vmware/harbor/src/ui/promgr" ) @@ -58,3 +59,20 @@ func (b *BaseController) Prepare() { } b.ProjectMgr = pm } + +//Init related objects/configurations for the API controllers +func Init() error { + //If chart repository is not enabled then directly return + if !config.WithChartMuseum() { + return nil + } + + chartCtl, err := initializeChartController() + if err != nil { + return err + } + + chartController = chartCtl + + return nil +} diff --git a/src/ui/api/chart_repository.go b/src/ui/api/chart_repository.go index 1d8d89634..86556b413 100644 --- a/src/ui/api/chart_repository.go +++ b/src/ui/api/chart_repository.go @@ -2,19 +2,23 @@ package api import ( "context" + "errors" "fmt" "net/http" "net/url" - "os" "strings" "github.com/vmware/harbor/src/chartserver" hlog "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/config" ) const ( - backendChartServerAddr = "BACKEND_CHART_SERVER" - namespaceParam = ":repo" + namespaceParam = ":repo" + defaultRepo = "library" + rootUploadingEndpoint = "/api/chartrepo/charts" + rootIndexEndpoint = "/chartrepo/index.yaml" + chartRepoHealthEndpoint = "/api/chartrepo/health" accessLevelPublic = iota accessLevelRead @@ -24,7 +28,7 @@ const ( ) //chartController is a singleton instance -var chartController = initializeChartController() +var chartController *chartserver.Controller //ChartRepositoryAPI provides related API handlers for the chart repository APIs type ChartRepositoryAPI struct { @@ -49,11 +53,20 @@ func (cra *ChartRepositoryAPI) Prepare() { // -/index.yaml // -/api/chartserver/health incomingURI := cra.Ctx.Request.RequestURI - if incomingURI != "/index.yaml" && incomingURI != "/api/chartserver/health" { + if incomingURI == rootUploadingEndpoint { + //Forward to the default repository + cra.namespace = defaultRepo + } + + if incomingURI != rootIndexEndpoint && + incomingURI != chartRepoHealthEndpoint { if !cra.requireNamespace(cra.namespace) { return } } + + //Rewrite URL path + cra.rewriteURLPath(cra.Ctx.Request) } //GetHealthStatus handles GET /api/chartserver/health @@ -63,11 +76,7 @@ func (cra *ChartRepositoryAPI) GetHealthStatus() { return } - //Override the request path to '/health' - req := cra.Ctx.Request - req.URL.Path = "/health" - - chartController.GetBaseHandler().GetHealthStatus(cra.Ctx.ResponseWriter, req) + chartController.GetBaseHandler().GetHealthStatus(cra.Ctx.ResponseWriter, cra.Ctx.Request) } //GetIndexByRepo handles GET /:repo/index.yaml @@ -146,6 +155,8 @@ func (cra *ChartRepositoryAPI) DeleteChartVersion() { //UploadChartVersion handles POST /api/:repo/charts func (cra *ChartRepositoryAPI) UploadChartVersion() { + hlog.Debugf("Header of request of uploading chart: %#v, content-len=%d", cra.Ctx.Request.Header, cra.Ctx.Request.ContentLength) + //Check access if !cra.requireAccess(cra.namespace, accessLevelWrite) { return @@ -164,6 +175,41 @@ func (cra *ChartRepositoryAPI) UploadChartProvFile() { chartController.GetManipulationHandler().UploadProvenanceFile(cra.Ctx.ResponseWriter, cra.Ctx.Request) } +//Rewrite the incoming URL with the right backend URL pattern +//Remove 'chartrepo' from the endpoints of manipulation API +//Remove 'chartrepo' from the endpoints of repository services +func (cra *ChartRepositoryAPI) rewriteURLPath(req *http.Request) { + incomingURLPath := req.RequestURI + + defer func() { + hlog.Debugf("Incoming URL '%s' is rewritten to '%s'", incomingURLPath, req.URL.String()) + }() + + //Health check endpoint + if incomingURLPath == chartRepoHealthEndpoint { + req.URL.Path = "/health" + return + } + + //Root uploading endpoint + if incomingURLPath == rootUploadingEndpoint { + req.URL.Path = strings.Replace(incomingURLPath, "chartrepo", defaultRepo, 1) + return + } + + //Repository endpoints + if strings.HasPrefix(incomingURLPath, "/chartrepo") { + req.URL.Path = strings.TrimPrefix(incomingURLPath, "/chartrepo") + return + } + + //API endpoints + if strings.HasPrefix(incomingURLPath, "/api/chartrepo") { + req.URL.Path = strings.Replace(incomingURLPath, "/chartrepo", "", 1) + return + } +} + //Check if there exists a valid namespace //Return true if it does //Return false if it does not @@ -242,19 +288,25 @@ func (cra *ChartRepositoryAPI) requireAccess(namespace string, accessLevel uint) } //Initialize the chart service controller -func initializeChartController() *chartserver.Controller { - addr := os.Getenv(backendChartServerAddr) +func initializeChartController() (*chartserver.Controller, error) { + addr, err := config.GetChartMuseumEndpoint() + if err != nil { + return nil, fmt.Errorf("Failed to get the endpoint URL of chart storage server: %s", err.Error()) + } + + addr = strings.TrimSuffix(addr, "/") url, err := url.Parse(addr) if err != nil { - hlog.Fatal("chart storage server is not correctly configured") + return nil, errors.New("Endpoint URL of chart storage server is malformed") } controller, err := chartserver.NewController(url) if err != nil { - hlog.Fatal("failed to initialize chart API controller") + return nil, errors.New("Failed to initialize chart API controller") } + hlog.Debugf("Chart storage server is set to %s", url.String()) hlog.Info("API controller for chart repository server is successfully initialized") - return controller + return controller, nil } diff --git a/src/ui/api/systeminfo.go b/src/ui/api/systeminfo.go index e7e9a9448..e19115acb 100644 --- a/src/ui/api/systeminfo.go +++ b/src/ui/api/systeminfo.go @@ -100,6 +100,7 @@ type GeneralInfo struct { ClairVulnStatus *models.ClairVulnerabilityStatus `json:"clair_vulnerability_status,omitempty"` RegistryStorageProviderName string `json:"registry_storage_provider_name"` ReadOnly bool `json:"read_only"` + WithChartMuseum bool `json:"with_chartmuseum"` } // validate for validating user if an admin. @@ -179,6 +180,7 @@ func (sia *SystemInfoAPI) GetGeneralInfo() { HarborVersion: harborVersion, RegistryStorageProviderName: utils.SafeCastString(cfg[common.RegistryStorageProviderName]), ReadOnly: config.ReadOnly(), + WithChartMuseum: config.WithChartMuseum(), } if info.WithClair { info.ClairVulnStatus = getClairVulnStatus() diff --git a/src/ui/config/config.go b/src/ui/config/config.go index 3ebba53e9..95da3a096 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -18,6 +18,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/json" + "errors" "fmt" "io/ioutil" "net/http" @@ -540,3 +541,31 @@ func ReadOnly() bool { } return utils.SafeCastBool(cfg[common.ReadOnly]) } + +// WithChartMuseum returns a bool to indicate if chartmuseum is deployed with Harbor. +func WithChartMuseum() bool { + cfg, err := mg.Get() + if err != nil { + log.Errorf("Failed to get 'with_chartmuseum' configuration with error: %s; return false as default", err.Error()) + return false + } + + return utils.SafeCastBool(cfg[common.WithChartMuseum]) +} + +// GetChartMuseumEndpoint returns the endpoint of the chartmuseum service +// otherwise an non nil error is returned +func GetChartMuseumEndpoint() (string, error) { + cfg, err := mg.Get() + if err != nil { + log.Errorf("Failed to get 'chart_repository_url' configuration with error: %s; return false as default", err.Error()) + return "", err + } + + chartEndpoint := strings.TrimSpace(utils.SafeCastString(cfg[common.ChartRepoURL])) + if len(chartEndpoint) == 0 { + return "", errors.New("empty chartmuseum endpoint") + } + + return chartEndpoint, nil +} diff --git a/src/ui/filter/mediatype.go b/src/ui/filter/mediatype.go index 8d2765faf..6874856f8 100644 --- a/src/ui/filter/mediatype.go +++ b/src/ui/filter/mediatype.go @@ -15,9 +15,12 @@ package filter import ( - beegoctx "github.com/astaxie/beego/context" "net/http" "strings" + + beegoctx "github.com/astaxie/beego/context" + + hlog "github.com/vmware/harbor/src/common/utils/log" ) //MediaTypeFilter filters the POST request, it returns 415 if the content type of the request @@ -34,6 +37,8 @@ func filterContentType(req *http.Request, resp http.ResponseWriter, mediaType .. } v := req.Header.Get("Content-Type") mimeType := strings.Split(v, ";")[0] + hlog.Debugf("Mimetype of incoming request %s: %s", req.RequestURI, mimeType) + for _, t := range mediaType { if t == mimeType { return diff --git a/src/ui/main.go b/src/ui/main.go index ef2b51850..4a4297103 100644 --- a/src/ui/main.go +++ b/src/ui/main.go @@ -113,6 +113,11 @@ func main() { log.Error(err) } + //Init API handler + if err := api.Init(); err != nil { + log.Fatalf("Failed to initialize API handlers with error: %s", err.Error()) + } + //Enable the policy scheduler here. scheduler.DefaultScheduler.Start() @@ -145,7 +150,7 @@ func main() { filter.Init() beego.InsertFilter("/*", beego.BeforeRouter, filter.SecurityFilter) beego.InsertFilter("/*", beego.BeforeRouter, filter.ReadonlyFilter) - beego.InsertFilter("/api/*", beego.BeforeRouter, filter.MediaTypeFilter("application/json")) + beego.InsertFilter("/api/*", beego.BeforeRouter, filter.MediaTypeFilter("application/json", "multipart/form-data", "application/octet-stream")) initRouters() diff --git a/src/ui/router.go b/src/ui/router.go index 4e184fc7a..24a3d2482 100644 --- a/src/ui/router.go +++ b/src/ui/router.go @@ -131,19 +131,23 @@ func initRouters() { beego.Router("/registryproxy/*", &controllers.RegistryProxy{}, "*:Handle") //APIs for chart repository - chartRepositoryAPIType := &api.ChartRepositoryAPI{} - beego.Router("/api/chartserver/health", chartRepositoryAPIType, "get:GetHealthStatus") - beego.Router("/api/:repo/charts", chartRepositoryAPIType, "get:ListCharts") - beego.Router("/api/:repo/charts/:name", chartRepositoryAPIType, "get:ListChartVersions") - beego.Router("/api/:repo/charts/:name/:version", chartRepositoryAPIType, "get:GetChartVersion") - beego.Router("/api/:repo/charts/:name/:version", chartRepositoryAPIType, "delete:DeleteChartVersion") - beego.Router("/api/:repo/charts", chartRepositoryAPIType, "post:UploadChartVersion") - beego.Router("/api/:repo/prov", chartRepositoryAPIType, "post:UploadChartProvFile") + if config.WithChartMuseum() { + //Charts are controlled under projects + chartRepositoryAPIType := &api.ChartRepositoryAPI{} + beego.Router("/api/chartrepo/health", chartRepositoryAPIType, "get:GetHealthStatus") + beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "get:ListCharts") + beego.Router("/api/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "get:ListChartVersions") + beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "get:GetChartVersion") + beego.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "delete:DeleteChartVersion") + beego.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "post:UploadChartVersion") + beego.Router("/api/chartrepo/:repo/prov", chartRepositoryAPIType, "post:UploadChartProvFile") + beego.Router("/api/chartrepo/charts", chartRepositoryAPIType, "post:UploadChartVersion") - //Repository services - beego.Router("/:repo/index.yaml", chartRepositoryAPIType, "get:GetIndexByRepo") - beego.Router("/index.yaml", chartRepositoryAPIType, "get:GetIndex") - beego.Router("/:repo/charts/:filename", chartRepositoryAPIType, "get:DownloadChart") + //Repository services + beego.Router("/chartrepo/:repo/index.yaml", chartRepositoryAPIType, "get:GetIndexByRepo") + beego.Router("/chartrepo/index.yaml", chartRepositoryAPIType, "get:GetIndex") + beego.Router("/chartrepo/:repo/charts/:filename", chartRepositoryAPIType, "get:DownloadChart") + } //Error pages beego.ErrorController(&controllers.ErrorController{}) diff --git a/tests/resources/Harbor-Util.robot b/tests/resources/Harbor-Util.robot index e638f6659..5bb5f427e 100644 --- a/tests/resources/Harbor-Util.robot +++ b/tests/resources/Harbor-Util.robot @@ -41,35 +41,35 @@ Install Harbor to Test Server Generate Certificate Authority For Chrome Up Harbor - [Arguments] ${with_notary}=true ${with_clair}=true - ${rc} ${output}= Run And Return Rc And Output make start -e NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} + [Arguments] ${with_notary}=true ${with_clair}=true ${with_chartmuseum}=true + ${rc} ${output}= Run And Return Rc And Output make start -e NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum} Log ${rc} Log ${output} Should Be Equal As Integers ${rc} 0 Down Harbor - [Arguments] ${with_notary}=true ${with_clair}=true - ${rc} ${output}= Run And Return Rc And Output echo "Y" | make down -e NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} + [Arguments] ${with_notary}=true ${with_clair}=true ${with_chartmuseum}=true + ${rc} ${output}= Run And Return Rc And Output echo "Y" | make down -e NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum} Log ${rc} Log ${output} Should Be Equal As Integers ${rc} 0 Package Harbor Offline - [Arguments] ${golang_image}=golang:${GOLANG_VERSION} ${clarity_image}=vmware/harbor-clarity-ui-builder:${CLAIR_BUILDER} ${with_notary}=true ${with_clair}=true ${with_migrator}=true + [Arguments] ${golang_image}=golang:${GOLANG_VERSION} ${clarity_image}=vmware/harbor-clarity-ui-builder:${CLAIR_BUILDER} ${with_notary}=true ${with_clair}=true ${with_migrator}=true ${with_chartmuseum}=true Log To Console \nStart Docker Daemon Start Docker Daemon Locally - Log To Console \n\nmake package_offline VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} HTTPPROXY= - ${rc} ${output}= Run And Return Rc And Output make package_offline VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} HTTPPROXY= + Log To Console \n\nmake package_offline VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} CHARTFLAG=${with_chartmuseum} HTTPPROXY= + ${rc} ${output}= Run And Return Rc And Output make package_offline VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} CHARTFLAG=${with_chartmuseum} HTTPPROXY= Log ${rc} Log ${output} Should Be Equal As Integers ${rc} 0 Package Harbor Online - [Arguments] ${golang_image}=golang:${GOLANG_VERSION} ${clarity_image}=vmware/harbor-clarity-ui-builder:${CLAIR_BUILDER} ${with_notary}=true ${with_clair}=true ${with_migrator}=true + [Arguments] ${golang_image}=golang:${GOLANG_VERSION} ${clarity_image}=vmware/harbor-clarity-ui-builder:${CLAIR_BUILDER} ${with_notary}=true ${with_clair}=true ${with_migrator}=true ${with_chartmuseum}=true Log To Console \nStart Docker Daemon Start Docker Daemon Locally - Log To Console \nmake package_online VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} HTTPPROXY= - ${rc} ${output}= Run And Return Rc And Output make package_online VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} HTTPPROXY= + Log To Console \nmake package_online VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} CHARTFLAG=${with_chartmuseum} HTTPPROXY= + ${rc} ${output}= Run And Return Rc And Output make package_online VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} UIVERSIONTAG=%{Harbor_UI_Version} GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} MIGRATORFLAG=${with_migrator} CHARTFLAG=${with_chartmuseum} HTTPPROXY= Log ${rc} Log ${output} Should Be Equal As Integers ${rc} 0 @@ -116,8 +116,8 @@ Enable Notary Client Log ${output} Prepare - [Arguments] ${with_notary}=true ${with_clair}=true - ${rc} ${output}= Run And Return Rc And Output make prepare -e NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} + [Arguments] ${with_notary}=true ${with_clair}=true ${with_chartmuseum}=true + ${rc} ${output}= Run And Return Rc And Output make prepare -e NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum} Log ${rc} Log ${output} Should Be Equal As Integers ${rc} 0 @@ -150,14 +150,14 @@ Prepare Cert Should Be Equal As Integers ${rc} 0 Compile and Up Harbor With Source Code - [Arguments] ${golang_image}=golang:${GOLANG_VERSION} ${clarity_image}=vmware/harbor-clarity-ui-builder:${CLAIR_BUILDER} ${with_notary}=true ${with_clair}=true + [Arguments] ${golang_image}=golang:${GOLANG_VERSION} ${clarity_image}=vmware/harbor-clarity-ui-builder:${CLAIR_BUILDER} ${with_notary}=true ${with_clair}=true ${with_chartmuseum}=true ${rc} ${output}= Run And Return Rc And Output docker pull ${clarity_image} Log ${output} Should Be Equal As Integers ${rc} 0 ${rc} ${output}= Run And Return Rc And Output docker pull ${golang_image} Log ${output} Should Be Equal As Integers ${rc} 0 - ${rc} ${output}= Run And Return Rc And Output make install GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} HTTPPROXY= + ${rc} ${output}= Run And Return Rc And Output make install GOBUILDIMAGE=${golang_image} COMPILETAG=compile_golangimage CLARITYIMAGE=${clarity_image} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum} HTTPPROXY= Log ${output} Should Be Equal As Integers ${rc} 0 Sleep 20