Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-22 02:05:41 +01:00
Code Issues Projects Releases Wiki Activity

remove chartmuseum backend (#18191)

Browse Source 
Harbor deprecates chartmuseum as of v2.8.0

Epic: https://github.com/goharbor/harbor/issues/17958

Discussion: https://github.com/goharbor/harbor/discussions/15057

Signed-off-by: Wang Yan <wangyan@vmware.com>
This commit is contained in:
Wang Yan 2023-02-16 18:11:05 +08:00 committed by GitHub
parent ec5afc3257
commit 738fde7d3b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
168 changed files with 154 additions and 8799 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/build-package.yml vendored
View File

@ -88,8 +88,8 @@ jobs:
else
build_base_params=" BUILD_BASE=true PUSHBASEIMAGE=true REGISTRYUSER=\"${{ secrets.DOCKER_HUB_USERNAME }}\" REGISTRYPASSWORD=\"${{ secrets.DOCKER_HUB_PASSWORD }}\""
fi
sudo make package_offline GOBUILDTAGS="include_oss include_gcs" BASEIMAGETAG=${Harbor_Build_Base_Tag} VERSIONTAG=${Harbor_Assets_Version} PKGVERSIONTAG=${Harbor_Package_Version} NOTARYFLAG=true CHARTFLAG=true TRIVYFLAG=true HTTPPROXY= ${build_base_params}
sudo make package_online GOBUILDTAGS="include_oss include_gcs" BASEIMAGETAG=${Harbor_Build_Base_Tag} VERSIONTAG=${Harbor_Assets_Version} PKGVERSIONTAG=${Harbor_Package_Version} NOTARYFLAG=true CHARTFLAG=true TRIVYFLAG=true HTTPPROXY= ${build_base_params}
sudo make package_offline GOBUILDTAGS="include_oss include_gcs" BASEIMAGETAG=${Harbor_Build_Base_Tag} VERSIONTAG=${Harbor_Assets_Version} PKGVERSIONTAG=${Harbor_Package_Version} NOTARYFLAG=true TRIVYFLAG=true HTTPPROXY= ${build_base_params}
sudo make package_online GOBUILDTAGS="include_oss include_gcs" BASEIMAGETAG=${Harbor_Build_Base_Tag} VERSIONTAG=${Harbor_Assets_Version} PKGVERSIONTAG=${Harbor_Package_Version} NOTARYFLAG=true TRIVYFLAG=true HTTPPROXY= ${build_base_params}
harbor_offline_build_bundle=$(basename harbor-offline-installer-*.tgz)
harbor_online_build_bundle=$(basename harbor-online-installer-*.tgz)
echo "Package name is: $harbor_offline_build_bundle"

24
Makefile
View File

@ -82,8 +82,6 @@ TRIVYFLAG=false
HTTPPROXY=
BUILDBIN=true
NPM_REGISTRY=https://registry.npmjs.org
# enable/disable chart repo supporting
CHARTFLAG=false
BUILDTARGET=build
GEN_TLS=
@ -94,7 +92,7 @@ VERSIONTAG=dev
BUILD_BASE=true
PUSHBASEIMAGE=false
BASEIMAGETAG=dev
BUILDBASETARGET=chartserver trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl exporter
BUILDBASETARGET=trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl exporter
IMAGENAMESPACE=goharbor
BASEIMAGENAMESPACE=goharbor
# #input true/false only
@ -112,17 +110,10 @@ NOTARYMIGRATEVERSION=v4.11.0
TRIVYVERSION=v0.37.2
TRIVYADAPTERVERSION=v0.30.7
# version of chartmuseum for pulling the source code
CHARTMUSEUM_SRC_TAG=v0.14.0
# version of chartmuseum
CHARTMUSEUMVERSION=$(CHARTMUSEUM_SRC_TAG)-redis
# version of registry for pulling the source code
REGISTRY_SRC_TAG=v2.8.0
# dependency binaries
CHARTURL=https://storage.googleapis.com/harbor-builds/bin/chartmuseum/release-${CHARTMUSEUMVERSION}/chartm
NOTARYURL=https://storage.googleapis.com/harbor-builds/bin/notary/release-${NOTARYVERSION}/binary-bundle.tgz
REGISTRYURL=https://storage.googleapis.com/harbor-builds/bin/registry/release-${REGISTRYVERSION}/registry
TRIVY_DOWNLOAD_URL=https://github.com/aquasecurity/trivy/releases/download/$(TRIVYVERSION)/trivy_$(TRIVYVERSION:v%=%)_Linux-64bit.tar.gz
@ -134,7 +125,6 @@ REGISTRY_VERSION: $(REGISTRYVERSION)
NOTARY_VERSION: $(NOTARYVERSION)
TRIVY_VERSION: $(TRIVYVERSION)
TRIVY_ADAPTER_VERSION: $(TRIVYADAPTERVERSION)
CHARTMUSEUM_VERSION: $(CHARTMUSEUMVERSION)
endef
# docker parameters
@ -216,10 +206,6 @@ endif
ifeq ($(TRIVYFLAG), true)
PREPARECMD_PARA+= --with-trivy
endif
# append chartmuseum parameters if set
ifeq ($(CHARTFLAG), true)
PREPARECMD_PARA+= --with-chartmuseum
endif
# makefile
MAKEFILEPATH_PHOTON=$(MAKEPATH)/photon
@ -234,7 +220,6 @@ DOCKERIMAGENAME_CORE=$(IMAGENAMESPACE)/harbor-core
DOCKERIMAGENAME_JOBSERVICE=$(IMAGENAMESPACE)/harbor-jobservice
DOCKERIMAGENAME_LOG=$(IMAGENAMESPACE)/harbor-log
DOCKERIMAGENAME_DB=$(IMAGENAMESPACE)/harbor-db
DOCKERIMAGENAME_CHART_SERVER=$(IMAGENAMESPACE)/chartmuseum-photon
DOCKERIMAGENAME_REGCTL=$(IMAGENAMESPACE)/harbor-registryctl
DOCKERIMAGENAME_EXPORTER=$(IMAGENAMESPACE)/harbor-exporter
@ -295,10 +280,6 @@ endif
ifeq ($(TRIVYFLAG), true)
DOCKERSAVE_PARA+= $(IMAGENAMESPACE)/trivy-adapter-photon:$(VERSIONTAG)
endif
# append chartmuseum parameters if set
ifeq ($(CHARTFLAG), true)
DOCKERSAVE_PARA+= $(DOCKERIMAGENAME_CHART_SERVER):$(VERSIONTAG)
endif
RUNCONTAINER=$(DOCKERCMD) run --rm -u $(shell id -u):$(shell id -g) -v $(BUILDPATH):$(BUILDPATH) -w $(BUILDPATH)
@ -430,9 +411,8 @@ build:
-e TRIVYVERSION=$(TRIVYVERSION) -e TRIVYADAPTERVERSION=$(TRIVYADAPTERVERSION) \
-e VERSIONTAG=$(VERSIONTAG) \
-e BUILDBIN=$(BUILDBIN) \
-e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e CHARTMUSEUM_SRC_TAG=$(CHARTMUSEUM_SRC_TAG) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER) \
-e NPM_REGISTRY=$(NPM_REGISTRY) -e BASEIMAGETAG=$(BASEIMAGETAG) -e IMAGENAMESPACE=$(IMAGENAMESPACE) -e BASEIMAGENAMESPACE=$(BASEIMAGENAMESPACE) \
-e CHARTURL=$(CHARTURL) -e NOTARYURL=$(NOTARYURL) -e REGISTRYURL=$(REGISTRYURL) \
-e NOTARYURL=$(NOTARYURL) -e REGISTRYURL=$(REGISTRYURL) \
-e TRIVY_DOWNLOAD_URL=$(TRIVY_DOWNLOAD_URL) -e TRIVY_ADAPTER_DOWNLOAD_URL=$(TRIVY_ADAPTER_DOWNLOAD_URL) \
-e PULL_BASE_FROM_DOCKERHUB=$(PULL_BASE_FROM_DOCKERHUB) -e BUILD_BASE=$(BUILD_BASE) \
-e REGISTRYUSER=$(REGISTRYUSER) -e REGISTRYPASSWORD=$(REGISTRYPASSWORD) \

5
api/v2.0/swagger.yaml
View File

@ -7646,11 +7646,6 @@ definitions:
x-nullable: true
x-omitempty: true
description: If the Harbor instance is deployed with nested notary.
with_chartmuseum:
type: boolean
x-nullable: true
x-omitempty: true
description: If the Harbor instance is deployed with nested chartmuseum.
registry_url:
type: string
x-nullable: true

7
make/harbor.yml.tmpl
View File

@ -56,7 +56,7 @@ data_volume: /data
# Uncomment storage_service setting If you want to using external storage
# storage_service:
# # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
# # of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signed certificate.
# # of registry's containers. This is usually needed when the user hosts a internal storage with self signed certificate.
# ca_bundle:
# # storage backend, default is filesystem, options include filesystem, azure, gcs, s3, swift and oss
@ -117,10 +117,6 @@ notification:
# Maximum retry count for webhook job
webhook_job_max_retry: 10
chart:
# Change the value of absolute_url to enabled can enable absolute url in chart
absolute_url: disabled
# Log configurations
log:
# options are debug, info, warning, error, fatal
@ -187,7 +183,6 @@ _version: 2.7.0
# # db_index 0 is for core, it's unchangeable
# registry_db_index: 1
# jobservice_db_index: 2
# chartmuseum_db_index: 3
# trivy_db_index: 5
# idle_timeout_seconds: 30

17
make/install.sh
View File

@ -9,8 +9,7 @@ set +o noglob
usage=$'Please set hostname and other necessary attributes in harbor.yml first. DO NOT use localhost or 127.0.0.1 for hostname, because Harbor needs to be accessed by external clients.
Please set --with-notary if needs enable Notary in Harbor, and set ui_url_protocol/ssl_cert/ssl_cert_key in harbor.yml bacause notary must run under https.
Please set --with-trivy if needs enable Trivy in Harbor
Please set --with-chartmuseum if needs enable Chartmuseum in Harbor'
Please set --with-trivy if needs enable Trivy in Harbor'
item=0
# notary is not enabled by default
@ -19,8 +18,6 @@ with_notary=$false
with_clair=$false
# trivy is not enabled by default
with_trivy=$false
# chartmuseum is not enabled by default
with_chartmuseum=$false
# flag to using docker compose v1 or v2, default would using v1 docker-compose
DOCKER_COMPOSE=docker-compose
@ -36,8 +33,6 @@ while [ $# -gt 0 ]; do
with_clair=true;;
--with-trivy)
with_trivy=true;;
--with-chartmuseum)
with_chartmuseum=true;;
*)
note "$usage"
exit 1;;
@ -83,10 +78,6 @@ if [ $with_trivy ]
then
prepare_para="${prepare_para} --with-trivy"
fi
if [ $with_chartmuseum ]
then
prepare_para="${prepare_para} --with-chartmuseum"
fi
./prepare $prepare_para
echo ""
@ -99,12 +90,6 @@ fi
echo ""
h2 "[Step $item]: starting Harbor ..."
if [ $with_chartmuseum ]
then
warn "
Chartmusuem will be deprecated as of Harbor v2.6.0 and start to be removed in v2.8.0 or later.
Please see discussion here for more details. https://github.com/goharbor/harbor/discussions/15057"
fi
if [ $with_notary ]
then
warn "

25
make/photon/Makefile
View File

@ -117,14 +117,6 @@ DOCKERFILEPATH_EXPORTER=$(DOCKERFILEPATH)/$(EXPORTER)
DOCKERFILENAME_EXPORTER=Dockerfile
DOCKERIMAGENAME_EXPORTER=$(IMAGENAMESPACE)/harbor-$(EXPORTER)
# for chart server (chartmuseum)
CHARTSERVER=chartserver
DOCKERFILEPATH_CHART_SERVER=$(DOCKERFILEPATH)/$(CHARTSERVER)
DOCKERFILENAME_CHART_SERVER=Dockerfile
CHART_SERVER_CODE_BASE=https://github.com/helm/chartmuseum.git
CHART_SERVER_MAIN_PATH=cmd/chartmuseum
CHART_SERVER_BIN_NAME=chartm
_build_prepare:
@$(call _build_base,$(PREPARE),$(DOCKERFILEPATH_PREPARE))
@echo "building prepare container for photon..."
@ -184,21 +176,6 @@ _build_trivy_adapter:
echo "Done." ; \
fi
_build_chart_server:
@if [ "$(CHARTFLAG)" = "true" ] ; then \
$(call _build_base,$(CHARTSERVER),$(DOCKERFILEPATH_CHART_SERVER)); \
if [ "$(BUILDBIN)" != "true" ] ; then \
rm -rf $(DOCKERFILEPATH_CHART_SERVER)/binary && mkdir -p $(DOCKERFILEPATH_CHART_SERVER)/binary && \
$(call _get_binary, $(CHARTURL), $(DOCKERFILEPATH_CHART_SERVER)/binary/chartm); \
else \
cd $(DOCKERFILEPATH_CHART_SERVER) && $(DOCKERFILEPATH_CHART_SERVER)/builder $(GOBUILDIMAGE) $(CHART_SERVER_CODE_BASE) $(CHARTMUSEUM_SRC_TAG) $(CHART_SERVER_MAIN_PATH) $(CHART_SERVER_BIN_NAME) && cd - ; \
fi ; \
echo "building chartmuseum container for photon..." ; \
$(DOCKERBUILD_WITH_PULL_PARA) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_CHART_SERVER)/$(DOCKERFILENAME_CHART_SERVER) -t $(DOCKERIMAGENAME_CHART_SERVER):$(VERSIONTAG) . ; \
rm -rf $(DOCKERFILEPATH_CHART_SERVER)/binary; \
echo "Done." ; \
fi
_build_nginx:
@$(call _build_base,$(NGINX),$(DOCKERFILEPATH_NGINX))
@echo "building nginx container for photon..."
@ -284,7 +261,7 @@ define _build_base
fi
endef
build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_trivy_adapter _build_redis _build_chart_server _compile_and_build_exporter
build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_trivy_adapter _build_redis _compile_and_build_exporter
@if [ -n "$(REGISTRYUSER)" ] && [ -n "$(REGISTRYPASSWORD)" ] ; then \
docker logout ; \
fi

23
make/photon/chartserver/Dockerfile
View File

@ -1,23 +0,0 @@
ARG harbor_base_image_version
ARG harbor_base_namespace
FROM ${harbor_base_namespace}/harbor-chartserver-base:${harbor_base_image_version}
COPY ./make/photon/chartserver/binary/chartm /home/chart/
COPY ./make/photon/chartserver/docker-entrypoint.sh /home/chart/
COPY ./make/photon/common/install_cert.sh /home/chart/
RUN chown -R chart:chart /etc/pki/tls/certs \
&& chown -R chart:chart /home/chart \
&& chmod u+x /home/chart/chartm \
&& chmod u+x /home/chart/docker-entrypoint.sh \
&& chmod u+x /home/chart/install_cert.sh
USER chart
WORKDIR /home/chart
ENTRYPOINT ["./docker-entrypoint.sh"]
VOLUME ["/chart_storage"]
HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS http://localhost:9999/health || curl -k -sS https://localhost:9443/health || exit 1

6
make/photon/chartserver/Dockerfile.base
View File

@ -1,6 +0,0 @@
FROM photon:4.0
RUN tdnf install -y shadow >>/dev/null\
&& tdnf clean all \
&& groupadd -r -g 10000 chart \
&& useradd --no-log-init -m -g 10000 -u 10000 chart

34
make/photon/chartserver/builder
View File

@ -1,34 +0,0 @@
#!/bin/bash
set +e
usage(){
echo "Usage: builder <golang image:version> <code path> <code release tag> <main.go path> <binary name>"
echo "e.g: builder golang:1.19.4 github.com/helm/chartmuseum v0.14.0 cmd/chartmuseum chartm"
exit 1
}
if [ $# != 5 ]; then
usage
fi
GOLANG_IMAGE="$1"
GIT_PATH="$2"
CODE_VERSION="$3"
MAIN_GO_PATH="$4"
BIN_NAME="$5"
set -e
cd `dirname $0`
cur=$PWD
mkdir -p binary
rm -rf binary/$BIN_NAME || true
cp compile.sh binary/
cp *.patch binary/
docker run -i --rm -v $cur/binary:/go/bin --name golang_code_builder $GOLANG_IMAGE /bin/bash /go/bin/compile.sh $GIT_PATH $CODE_VERSION $MAIN_GO_PATH $BIN_NAME
#Clear
#docker rm -f golang_code_builder

36
make/photon/chartserver/compile.sh
View File

@ -1,36 +0,0 @@
#!/bin/bash
set +e
usage(){
echo "Usage: compile.sh <code path> <code tag> <main.go path> <binary name>"
echo "e.g: compile.sh github.com/helm/chartmuseum v0.14.0 cmd/chartmuseum chartm"
exit 1
}
if [ $# != 4 ]; then
usage
fi
GIT_PATH="$1"
VERSION="$2"
MAIN_GO_PATH="$3"
BIN_NAME="$4"
#Get the source code
git clone $GIT_PATH src_code
ls
SRC_PATH=$(pwd)/src_code
set -e
#Checkout the released tag branch
cd $SRC_PATH
git checkout tags/$VERSION -b $VERSION
#Patch
for p in $(ls /go/bin/*.patch); do
git apply $p || exit /b 1
done
#Compile
cd $SRC_PATH/$MAIN_GO_PATH && go build -a -o $BIN_NAME
mv $BIN_NAME /go/bin/

8
make/photon/chartserver/docker-entrypoint.sh
View File

@ -1,8 +0,0 @@
#!/bin/bash
set -e
/home/chart/install_cert.sh
#Start the server process
exec /home/chart/chartm

77
make/photon/chartserver/redis.patch
View File

@ -1,77 +0,0 @@
diff --git a/cmd/chartmuseum/main.go b/cmd/chartmuseum/main.go
index 8fd7349..eb67ddf 100644
--- a/cmd/chartmuseum/main.go
+++ b/cmd/chartmuseum/main.go
@@ -282,12 +282,23 @@ func storeFromConfig(conf *config.Config) cache.Store {
switch cacheFlag {
case "redis":
store = redisCacheFromConfig(conf)
+ case "redis_sentinel":
+ store = redisSentinelCacheFromConfig(conf)
default:
crash("Unsupported cache store: ", cacheFlag)
}
return store
}
+func redisSentinelCacheFromConfig(conf *config.Config) cache.Store {
+ crashIfConfigMissingVars(conf, []string{"cache.redis.addr", "cache.redis.mastername"})
+ return cache.Store(cache.NewRedisSentinelStore(
+ conf.GetString("cache.redis.mastername"),
+ strings.Split(conf.GetString("cache.redis.addr"), ","),
+ conf.GetString("cache.redis.password"),
+ conf.GetInt("cache.redis.db"),
+ ))
+}
func redisCacheFromConfig(conf *config.Config) cache.Store {
crashIfConfigMissingVars(conf, []string{"cache.redis.addr"})
diff --git a/pkg/cache/redis_sentinel.go b/pkg/cache/redis_sentinel.go
new file mode 100644
index 0000000..0c73427
--- /dev/null
+++ b/pkg/cache/redis_sentinel.go
@@ -0,0 +1,18 @@
+package cache
+
+import (
+ "github.com/go-redis/redis"
+)
+
+// NewRedisStore creates a new RedisStore
+func NewRedisSentinelStore(masterName string, sentinelAddrs []string, password string, db int) *RedisStore {
+ store := &RedisStore{}
+ redisClientOptions := &redis.FailoverOptions{
+ MasterName: masterName,
+ SentinelAddrs: sentinelAddrs,
+ Password: password,
+ DB: db,
+ }
+ store.Client = redis.NewFailoverClient(redisClientOptions)
+ return store
+}
diff --git a/pkg/config/vars.go b/pkg/config/vars.go
index 638eb49..4f036c9 100644
--- a/pkg/config/vars.go
+++ b/pkg/config/vars.go
@@ -246,10 +246,19 @@ var configVars = map[string]configVar{
Default: "",
CLIFlag: cli.StringFlag{
Name: "cache-redis-addr",
- Usage: "address of Redis service (host:port)",
+ Usage: "address of Redis service (host:port), addresses of Redis+Sentinel service (host1:port1,host2:port2)",
EnvVar: "CACHE_REDIS_ADDR",
},
},
+ "cache.redis.mastername": {
+ Type: stringType,
+ Default: "",
+ CLIFlag: cli.StringFlag{
+ Name: "cache-redis-mastername",
+ Usage: "address of Redis+Sentinel mastername",
+ EnvVar: "CACHE_REDIS_MASTERNAME",
+ },
+ },
"cache.redis.password": {
Type: stringType,
Default: "",

13
make/photon/prepare/commands/prepare.py
View File

@ -15,7 +15,6 @@ from utils.registry_ctl import prepare_registry_ctl
from utils.core import prepare_core
from utils.notary import prepare_notary
from utils.log import prepare_log_configs
from utils.chart import prepare_chartmuseum
from utils.docker_compose import prepare_docker_compose
from utils.nginx import prepare_nginx, nginx_confd_dir
from utils.redis import prepare_redis
@ -30,11 +29,10 @@ old_private_key_pem_path, old_crt_path)
@click.option('--conf', default=input_config_path, help="the path of Harbor configuration file")
@click.option('--with-notary', is_flag=True, help="the Harbor instance is to be deployed with notary")
@click.option('--with-trivy', is_flag=True, help="the Harbor instance is to be deployed with Trivy")
@click.option('--with-chartmuseum', is_flag=True, help="the Harbor instance is to be deployed with chart repository supporting")
def prepare(conf, with_notary, with_trivy, with_chartmuseum):
def prepare(conf, with_notary, with_trivy):
delfile(config_dir)
config_dict = parse_yaml_config(conf, with_notary=with_notary, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
config_dict = parse_yaml_config(conf, with_notary=with_notary, with_trivy=with_trivy)
try:
validate(config_dict, notary_mode=with_notary)
except Exception as e:
@ -45,7 +43,7 @@ def prepare(conf, with_notary, with_trivy, with_chartmuseum):
prepare_portal(config_dict)
prepare_log_configs(config_dict)
prepare_nginx(config_dict)
prepare_core(config_dict, with_notary=with_notary, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
prepare_core(config_dict, with_notary=with_notary, with_trivy=with_trivy)
prepare_registry(config_dict)
prepare_registry_ctl(config_dict)
prepare_db(config_dict)
@ -72,7 +70,4 @@ def prepare(conf, with_notary, with_trivy, with_chartmuseum):
if with_trivy:
prepare_trivy_adapter(config_dict)
if with_chartmuseum:
prepare_chartmuseum(config_dict)
prepare_docker_compose(config_dict, with_trivy, with_notary, with_chartmuseum)
prepare_docker_compose(config_dict, with_trivy, with_notary)

1
make/photon/prepare/g.py
View File

@ -57,7 +57,6 @@ INTERNAL_NO_PROXY_DN = {
'jobservice',
'registry',
'registryctl',
'chartmuseum',
'notary-server',
'notary-signer',
'trivy-adapter',

2
make/photon/prepare/migrations/version_1_10_0/harbor.yml.jinja
View File

@ -287,7 +287,6 @@ external_redis:
# db_index 0 is for core, it's unchangeable
registry_db_index: {{ external_redis.registry_db_index }}
jobservice_db_index: {{ external_redis.jobservice_db_index }}
chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }}
clair_db_index: 4
{% else %}
# Umcomments external_redis if using external Redis server
@ -298,7 +297,6 @@ external_redis:
# # db_index 0 is for core, it's unchangeable
# registry_db_index: 1
# jobservice_db_index: 2
# chartmuseum_db_index: 3
# clair_db_index: 4
{% endif %}

2
make/photon/prepare/migrations/version_1_9_0/harbor.yml.jinja
View File

@ -226,7 +226,6 @@ external_redis:
# db_index 0 is for core, it's unchangeable
registry_db_index: {{ external_redis.registry_db_index }}
jobservice_db_index: {{ external_redis.jobservice_db_index }}
chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }}
{% else %}
# Umcomments external_redis if using external Redis server
# external_redis:
@ -236,7 +235,6 @@ external_redis:
# # db_index 0 is for core, it's unchangeable
# registry_db_index: 1
# jobservice_db_index: 2
# chartmuseum_db_index: 3
{% endif %}
{% if uaa is defined %}

2
make/photon/prepare/migrations/version_2_0_0/harbor.yml.jinja
View File

@ -330,7 +330,6 @@ external_redis:
# db_index 0 is for core, it's unchangeable
registry_db_index: {{ external_redis.registry_db_index }}
jobservice_db_index: {{ external_redis.jobservice_db_index }}
chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }}
clair_db_index: {{ external_redis.clair_db_index }}
trivy_db_index: 5
idle_timeout_seconds: 30
@ -348,7 +347,6 @@ external_redis:
# # db_index 0 is for core, it's unchangeable
# registry_db_index: 1
# jobservice_db_index: 2
# chartmuseum_db_index: 3
# clair_db_index: 4
# trivy_db_index: 5
# idle_timeout_seconds: 30

2
make/photon/prepare/migrations/version_2_1_0/harbor.yml.jinja
View File

@ -380,7 +380,6 @@ external_redis:
# db_index 0 is for core, it's unchangeable
registry_db_index: {{ external_redis.registry_db_index }}
jobservice_db_index: {{ external_redis.jobservice_db_index }}
chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }}
clair_db_index: {{ external_redis.clair_db_index }}
trivy_db_index: 5
idle_timeout_seconds: 30
@ -398,7 +397,6 @@ external_redis:
# # db_index 0 is for core, it's unchangeable
# registry_db_index: 1
# jobservice_db_index: 2
# chartmuseum_db_index: 3
# clair_db_index: 4
# trivy_db_index: 5
# idle_timeout_seconds: 30

2
make/photon/prepare/migrations/version_2_2_0/harbor.yml.jinja
View File

@ -357,7 +357,6 @@ external_redis:
# db_index 0 is for core, it's unchangeable
registry_db_index: {{ external_redis.registry_db_index }}
jobservice_db_index: {{ external_redis.jobservice_db_index }}
chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }}
trivy_db_index: 5
idle_timeout_seconds: 30
{% else %}
@ -374,7 +373,6 @@ external_redis:
# # db_index 0 is for core, it's unchangeable
# registry_db_index: 1
# jobservice_db_index: 2
# chartmuseum_db_index: 3
# trivy_db_index: 5
# idle_timeout_seconds: 30
{% endif %}

2
make/photon/prepare/migrations/version_2_3_0/harbor.yml.jinja
View File

@ -357,7 +357,6 @@ external_redis:
# db_index 0 is for core, it's unchangeable
registry_db_index: {{ external_redis.registry_db_index }}
jobservice_db_index: {{ external_redis.jobservice_db_index }}
chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }}
trivy_db_index: 5
idle_timeout_seconds: 30
{% else %}
@ -374,7 +373,6 @@ external_redis:
# # db_index 0 is for core, it's unchangeable
# registry_db_index: 1
# jobservice_db_index: 2
# chartmuseum_db_index: 3
# trivy_db_index: 5
# idle_timeout_seconds: 30
{% endif %}

2
make/photon/prepare/migrations/version_2_4_0/harbor.yml.jinja
View File

@ -363,7 +363,6 @@ external_redis:
# db_index 0 is for core, it's unchangeable
registry_db_index: {{ external_redis.registry_db_index }}
jobservice_db_index: {{ external_redis.jobservice_db_index }}
chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }}
trivy_db_index: 5
idle_timeout_seconds: 30
{% else %}
@ -380,7 +379,6 @@ external_redis:
# # db_index 0 is for core, it's unchangeable
# registry_db_index: 1
# jobservice_db_index: 2
# chartmuseum_db_index: 3
# trivy_db_index: 5
# idle_timeout_seconds: 30
{% endif %}

2
make/photon/prepare/migrations/version_2_5_0/harbor.yml.jinja
View File

@ -377,7 +377,6 @@ external_redis:
# db_index 0 is for core, it's unchangeable
registry_db_index: {{ external_redis.registry_db_index }}
jobservice_db_index: {{ external_redis.jobservice_db_index }}
chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }}
trivy_db_index: 5
idle_timeout_seconds: 30
{% else %}
@ -394,7 +393,6 @@ external_redis:
# # db_index 0 is for core, it's unchangeable
# registry_db_index: 1
# jobservice_db_index: 2
# chartmuseum_db_index: 3
# trivy_db_index: 5
# idle_timeout_seconds: 30
{% endif %}

2
make/photon/prepare/migrations/version_2_6_0/harbor.yml.jinja
View File

@ -377,7 +377,6 @@ external_redis:
# db_index 0 is for core, it's unchangeable
registry_db_index: {{ external_redis.registry_db_index }}
jobservice_db_index: {{ external_redis.jobservice_db_index }}
chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }}
trivy_db_index: 5
idle_timeout_seconds: 30
{% else %}
@ -394,7 +393,6 @@ external_redis:
# # db_index 0 is for core, it's unchangeable
# registry_db_index: 1
# jobservice_db_index: 2
# chartmuseum_db_index: 3
# trivy_db_index: 5
# idle_timeout_seconds: 30
{% endif %}

2
make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja
View File

@ -384,7 +384,6 @@ external_redis:
# db_index 0 is for core, it's unchangeable
registry_db_index: {{ external_redis.registry_db_index }}
jobservice_db_index: {{ external_redis.jobservice_db_index }}
chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }}
trivy_db_index: 5
idle_timeout_seconds: 30
{% else %}
@ -401,7 +400,6 @@ external_redis:
# # db_index 0 is for core, it's unchangeable
# registry_db_index: 1
# jobservice_db_index: 2
# chartmuseum_db_index: 3
# trivy_db_index: 5
# idle_timeout_seconds: 30
{% endif %}

7
make/photon/prepare/models.py
View File

@ -29,11 +29,6 @@ class InternalTLS:
'notary_server.crt', 'notary_server.key'
}
chart_museum_filename = {
'chartmuseum.crt',
'chartmuseum.key'
}
db_certs_filename = {
'harbor_db.crt', 'harbor_db.key'
}
@ -47,8 +42,6 @@ class InternalTLS:
self.required_filenames = self.harbor_certs_filename
if kwargs.get('with_notary'):
self.required_filenames.update(self.notary_certs_filename)
if kwargs.get('with_chartmuseum'):
self.required_filenames.update(self.chart_museum_filename)
if kwargs.get('with_trivy'):
self.required_filenames.update(self.trivy_certs_filename)
if not kwargs.get('external_database'):

11
make/photon/prepare/scripts/gencert.sh
View File

@ -115,17 +115,6 @@ echo subjectAltName = DNS.1:notary-server > extfile.cnf
openssl x509 -req -days $DAYS -sha256 -in notary_server.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -extfile extfile.cnf -out notary_server.crt
# generate chartmuseum key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout chartmuseum.key \
-out chartmuseum.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=chartmuseum"
# sign chartmuseum csr with CA certificate and key
echo subjectAltName = DNS.1:chartmuseum > extfile.cnf
openssl x509 -req -days $DAYS -sha256 -in chartmuseum.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -extfile extfile.cnf -out chartmuseum.crt
# generate harbor_db key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout harbor_db.key \

59
make/photon/prepare/templates/chartserver/env.jinja
View File

@ -1,59 +0,0 @@
## Settings should be set
{%if internal_tls.enabled %}
PORT=9443
TLS_CERT=/etc/harbor/ssl/chartmuseum.crt
TLS_KEY=/etc/harbor/ssl/chartmuseum.key
# Uncomment if need mTLS
# TLS_CA_CERT=/etc/harbor/ssl/harbor_internal_ca.crt
{% else %}
PORT=9999
{% endif %}
# Only support redis now. If redis is setup, then enable cache
CACHE={{cache_store}}
{% if cache_redis_mastername %}
CACHE_REDIS_ADDR={{cache_redis_addr}}
CACHE_REDIS_MASTERNAME={{cache_redis_mastername}}
CACHE_REDIS_PASSWORD={{cache_redis_password}}
CACHE_REDIS_DB={{cache_redis_db_index}}
{% else %}
CACHE_REDIS_ADDR={{cache_redis_addr}}
CACHE_REDIS_PASSWORD={{cache_redis_password}}
CACHE_REDIS_DB={{cache_redis_db_index}}
{% endif %}
# Credential for internal communication
BASIC_AUTH_USER=chart_controller
BASIC_AUTH_PASS={{core_secret}}
# Multiple tenants
# Must be set with 1 to support project namespace
DEPTH=1
# Backend storage driver: e.g. "local", "amazon", "google" etc.
STORAGE={{storage_driver}}
{% if storage_driver == "amazon" %}
AWS_SDK_LOAD_CONFIG=1
{% endif %}
# Storage driver settings
{{all_storage_driver_configs}}
## Settings with default values. Just put here for future changes
DEBUG=false
LOG_JSON=true
DISABLE_METRICS=false
DISABLE_API=false
DISABLE_STATEFILES=false
ALLOW_OVERWRITE=true
{% if chart_absolute_url %}
CHART_URL={{public_url}}/chartrepo
{% else %}
CHART_URL=
{% endif %}
AUTH_ANONYMOUS_GET=false
CONTEXT_PATH=
INDEX_LIMIT=0
MAX_STORAGE_OBJECTS=0
MAX_UPLOAD_SIZE=20971520
CHART_POST_FORM_FIELD_NAME=chart
PROV_POST_FORM_FIELD_NAME=prov
STORAGE_TIMESTAMP_TOLERANCE=1s

3
make/photon/prepare/templates/core/env.jinja
View File

@ -2,7 +2,6 @@ CONFIG_PATH=/etc/core/app.conf
UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem
_REDIS_URL_CORE={{redis_url_core}}
SYNC_QUOTA=true
CHART_CACHE_DRIVER={{chart_cache_driver}}
_REDIS_URL_REG={{redis_url_reg}}
LOG_LEVEL={{log_level}}
@ -35,9 +34,7 @@ NOTARY_URL={{notary_url}}
REGISTRY_STORAGE_PROVIDER_NAME={{storage_provider_name}}
READ_ONLY=false
RELOAD_KEY={{reload_key}}
CHART_REPOSITORY_URL={{chart_repository_url}}
REGISTRY_CONTROLLER_URL={{registry_controller_url}}
WITH_CHARTMUSEUM={{with_chartmuseum}}
REGISTRY_CREDENTIAL_USERNAME={{registry_username}}
REGISTRY_CREDENTIAL_PASSWORD={{registry_password}}
CSRF_KEY={{csrf_key}}

58
make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja
View File

@ -186,11 +186,6 @@ services:
harbor:
{% if with_notary %}
harbor-notary:
{% endif %}
{% if with_chartmuseum %}
harbor-chartmuseum:
aliases:
- harbor-core
{% endif %}
depends_on:
- log
@ -291,11 +286,6 @@ services:
- {{data_volume}}/redis:/var/lib/redis
networks:
harbor:
{% if with_chartmuseum %}
harbor-chartmuseum:
aliases:
- redis
{% endif %}
depends_on:
- log
logging:
@ -477,49 +467,6 @@ services:
env_file:
./common/config/trivy-adapter/env
{% endif %}
{% if with_chartmuseum %}
chartmuseum:
container_name: chartmuseum
image: goharbor/chartmuseum-photon:{{chartmuseum_version}}
restart: always
cap_drop:
- ALL
cap_add:
- CHOWN
- DAC_OVERRIDE
- SETGID
- SETUID
networks:
- harbor-chartmuseum
depends_on:
- log
volumes:
- {{data_volume}}/chart_storage:/chart_storage:z
- ./common/config/chartserver:/etc/chartserver:z
- type: bind
source: ./common/config/shared/trust-certificates
target: /harbor_cust_cert
{%if internal_tls.enabled %}
- type: bind
source: {{internal_tls.chartmuseum_crt_path}}
target: /etc/harbor/ssl/chartmuseum.crt
- type: bind
source: {{internal_tls.chartmuseum_key_path}}
target: /etc/harbor/ssl/chartmuseum.key
{% endif %}
{% if gcs_keyfile %}
- type: bind
source: {{gcs_keyfile}}
target: /etc/chartserver/gcs.key
{% endif %}
logging:
driver: "syslog"
options:
syslog-address: "tcp://localhost:1514"
tag: "chartmuseum"
env_file:
./common/config/chartserver/env
{% endif %}
{% if metric.enabled %}
exporter:
image: goharbor/harbor-exporter:{{version}}
@ -553,7 +500,4 @@ networks:
notary-sig:
external: false
{% endif %}
{% if with_chartmuseum %}
harbor-chartmuseum:
external: false
{% endif %}

22
make/photon/prepare/templates/nginx/nginx.http.conf.jinja
View File

@ -125,28 +125,6 @@ http {
proxy_request_buffering off;
}
location /chartrepo/ {
{% if internal_tls.enabled %}
proxy_pass https://core/chartrepo/;
proxy_ssl_certificate /etc/harbor/tls/proxy.crt;
proxy_ssl_certificate_key /etc/harbor/tls/proxy.key;
proxy_ssl_trusted_certificate /harbor_cust_cert/harbor_internal_ca.crt;
proxy_ssl_verify_depth 2;
proxy_ssl_verify on;
proxy_ssl_session_reuse on;
{% else %}
proxy_pass http://core/chartrepo/;
{% endif %}
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $x_forwarded_proto;
proxy_buffering off;
proxy_request_buffering off;
}
location /v1/ {
return 404;
}

24
make/photon/prepare/templates/nginx/nginx.https.conf.jinja
View File

@ -149,30 +149,6 @@ http {
proxy_request_buffering off;
}
location /chartrepo/ {
{% if internal_tls.enabled %}
proxy_pass https://core/chartrepo/;
proxy_ssl_certificate /etc/harbor/tls/proxy.crt;
proxy_ssl_certificate_key /etc/harbor/tls/proxy.key;
proxy_ssl_trusted_certificate /harbor_cust_cert/harbor_internal_ca.crt;
proxy_ssl_verify_depth 2;
proxy_ssl_verify on;
proxy_ssl_session_reuse on;
{% else %}
proxy_pass http://core/chartrepo/;
{% endif %}
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $x_forwarded_proto;
proxy_cookie_path / "/; Secure";
proxy_buffering off;
proxy_request_buffering off;
}
location /v1/ {
return 404;
}

122
make/photon/prepare/utils/chart.py
View File

@ -1,122 +0,0 @@
import os
from urllib.parse import urlsplit
from g import templates_dir, config_dir, data_dir, DEFAULT_UID, DEFAULT_GID
from .jinja import render_jinja
from .misc import prepare_dir
chart_museum_temp_dir = os.path.join(templates_dir, "chartserver")
chart_museum_env_temp = os.path.join(chart_museum_temp_dir, "env.jinja")
chart_museum_config_dir = os.path.join(config_dir, "chartserver")
chart_museum_env = os.path.join(config_dir, "chartserver", "env")
chart_museum_data_dir = os.path.join(data_dir, 'chart_storage')
def parse_redis(redis_url_chart):
u = urlsplit(redis_url_chart)
if not u.scheme or u.scheme == 'redis':
return {
'cache_store': 'redis',
'cache_redis_addr': u.netloc.split('@')[-1],
'cache_redis_password': u.password or '',
'cache_redis_db_index': u.path and int(u.path[1:]) or 0,
}
elif u.scheme == 'redis+sentinel':
return {
'cache_store': 'redis_sentinel',
'cache_redis_mastername': u.path.split('/')[1],
'cache_redis_addr': u.netloc.split('@')[-1],
'cache_redis_password': u.password or '',
'cache_redis_db_index': len(u.path.split('/')) == 3 and int(u.path.split('/')[2]) or 0,
}
else:
raise Exception('bad redis url for chart:' + redis_url_chart)
def prepare_chartmuseum(config_dict):
storage_provider_name = config_dict['storage_provider_name']
storage_provider_config_map = config_dict['storage_provider_config']
prepare_dir(chart_museum_data_dir, uid=DEFAULT_UID, gid=DEFAULT_GID)
prepare_dir(chart_museum_config_dir)
# process redis info
cache_redis_ops = parse_redis(config_dict['redis_url_chart'])
# process storage info
#default using local file system
storage_driver = "local"
# storage provider configurations
# please be aware that, we do not check the validations of the values for the specified keys
# convert the configs to config map
storage_provider_config_options = []
if storage_provider_name == 's3':
# aws s3 storage
storage_driver = "amazon"
storage_provider_config_options.append("STORAGE_AMAZON_BUCKET=%s" % (storage_provider_config_map.get("bucket") or '') )
storage_provider_config_options.append("STORAGE_AMAZON_PREFIX=%s" % (storage_provider_config_map.get("rootdirectory") or '') )
storage_provider_config_options.append("STORAGE_AMAZON_REGION=%s" % (storage_provider_config_map.get("region") or '') )
storage_provider_config_options.append("STORAGE_AMAZON_ENDPOINT=%s" % (storage_provider_config_map.get("regionendpoint") or '') )
storage_provider_config_options.append("AWS_ACCESS_KEY_ID=%s" % (storage_provider_config_map.get("accesskey") or '') )
storage_provider_config_options.append("AWS_SECRET_ACCESS_KEY=%s" % (storage_provider_config_map.get("secretkey") or '') )
elif storage_provider_name == 'gcs':
# google cloud storage
storage_driver = "google"
storage_provider_config_options.append("STORAGE_GOOGLE_BUCKET=%s" % ( storage_provider_config_map.get("bucket") or '') )
storage_provider_config_options.append("STORAGE_GOOGLE_PREFIX=%s" % ( storage_provider_config_map.get("rootdirectory") or '') )
if storage_provider_config_map.get("keyfile"):
storage_provider_config_options.append('GOOGLE_APPLICATION_CREDENTIALS=%s' % '/etc/chartserver/gcs.key')
elif storage_provider_name == 'azure':
# azure storage
storage_driver = "microsoft"
storage_provider_config_options.append("STORAGE_MICROSOFT_CONTAINER=%s" % ( storage_provider_config_map.get("container") or '') )
storage_provider_config_options.append("AZURE_STORAGE_ACCOUNT=%s" % ( storage_provider_config_map.get("accountname") or '') )
storage_provider_config_options.append("AZURE_STORAGE_ACCESS_KEY=%s" % ( storage_provider_config_map.get("accountkey") or '') )
storage_provider_config_options.append("STORAGE_MICROSOFT_PREFIX=/azure/harbor/charts")
elif storage_provider_name == 'swift':
# open stack swift
storage_driver = "openstack"
storage_provider_config_options.append("STORAGE_OPENSTACK_CONTAINER=%s" % ( storage_provider_config_map.get("container") or '') )
storage_provider_config_options.append("STORAGE_OPENSTACK_PREFIX=%s" % ( storage_provider_config_map.get("rootdirectory") or '') )
storage_provider_config_options.append("STORAGE_OPENSTACK_REGION=%s" % ( storage_provider_config_map.get("region") or '') )
storage_provider_config_options.append("OS_AUTH_URL=%s" % ( storage_provider_config_map.get("authurl") or '') )
storage_provider_config_options.append("OS_USERNAME=%s" % ( storage_provider_config_map.get("username") or '') )
storage_provider_config_options.append("OS_PASSWORD=%s" % ( storage_provider_config_map.get("password") or '') )
storage_provider_config_options.append("OS_PROJECT_ID=%s" % ( storage_provider_config_map.get("tenantid") or '') )
storage_provider_config_options.append("OS_PROJECT_NAME=%s" % ( storage_provider_config_map.get("tenant") or '') )
storage_provider_config_options.append("OS_DOMAIN_ID=%s" % ( storage_provider_config_map.get("domainid") or '') )
storage_provider_config_options.append("OS_DOMAIN_NAME=%s" % ( storage_provider_config_map.get("domain") or '') )
elif storage_provider_name == 'oss':
# aliyun OSS
storage_driver = "alibaba"
bucket = storage_provider_config_map.get("bucket") or ''
endpoint = storage_provider_config_map.get("endpoint") or ''
if endpoint.startswith(bucket + "."):
endpoint = endpoint.replace(bucket + ".", "")
storage_provider_config_options.append("STORAGE_ALIBABA_BUCKET=%s" % bucket )
storage_provider_config_options.append("STORAGE_ALIBABA_ENDPOINT=%s" % endpoint )
storage_provider_config_options.append("STORAGE_ALIBABA_PREFIX=%s" % ( storage_provider_config_map.get("rootdirectory") or '') )
storage_provider_config_options.append(
"ALIBABA_CLOUD_ACCESS_KEY_ID=%s" % (storage_provider_config_map.get("accesskeyid") or ''))
storage_provider_config_options.append(
"ALIBABA_CLOUD_ACCESS_KEY_SECRET=%s" % (storage_provider_config_map.get("accesskeysecret") or ''))
else:
# use local file system
storage_provider_config_options.append("STORAGE_LOCAL_ROOTDIR=/chart_storage")
# generate storage provider configuration
all_storage_provider_configs = ('\n').join(storage_provider_config_options)
render_jinja(
chart_museum_env_temp,
chart_museum_env,
storage_driver=storage_driver,
all_storage_driver_configs=all_storage_provider_configs,
public_url=config_dict['public_url'],
chart_absolute_url=config_dict['chart_absolute_url'],
internal_tls=config_dict['internal_tls'],
**cache_redis_ops)

14
make/photon/prepare/utils/configs.py
View File

@ -97,7 +97,7 @@ def parse_versions():
return versions
def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseum):
def parse_yaml_config(config_file_path, with_notary, with_trivy):
'''
:param configs: config_parser object
:returns: dict of configs
@ -116,7 +116,6 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
'jobservice_url': 'http://jobservice:8080',
'trivy_adapter_url': 'http://trivy-adapter:8080',
'notary_url': 'http://notary-server:4443',
'chart_repository_url': 'http://chartmuseum:9999'
}
config_dict['hostname'] = configs["hostname"]
@ -236,13 +235,6 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
config_dict['trivy_insecure'] = trivy_configs.get("insecure") or False
config_dict['trivy_timeout'] = trivy_configs.get("timeout") or '5m0s'
# Chart configs
chart_configs = configs.get("chart") or {}
if chart_configs.get('absolute_url') == 'enabled':
config_dict['chart_absolute_url'] = True
else:
config_dict['chart_absolute_url'] = False
# jobservice config
js_config = configs.get('jobservice') or {}
config_dict['max_job_workers'] = js_config["max_job_workers"]
@ -333,7 +325,6 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
configs['data_volume'],
with_notary=with_notary,
with_trivy=with_trivy,
with_chartmuseum=with_chartmuseum,
external_database=config_dict['external_database'])
else:
config_dict['internal_tls'] = InternalTLS()
@ -359,7 +350,6 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
config_dict['jobservice_url'] = 'https://jobservice:8443'
config_dict['trivy_adapter_url'] = 'https://trivy-adapter:8443'
# config_dict['notary_url'] = 'http://notary-server:4443'
config_dict['chart_repository_url'] = 'https://chartmuseum:9443'
# purge upload configs
purge_upload_config = configs.get('upload_purging')
@ -450,7 +440,6 @@ def get_redis_configs(external_redis=None, with_trivy=True):
'password': '',
'registry_db_index': 1,
'jobservice_db_index': 2,
'chartmuseum_db_index': 3,
'trivy_db_index': 5,
'idle_timeout_seconds': 30,
}
@ -459,7 +448,6 @@ def get_redis_configs(external_redis=None, with_trivy=True):
redis.update({key: value for (key, value) in external_redis.items() if value})
configs['redis_url_core'] = get_redis_url(0, redis)
configs['redis_url_chart'] = get_redis_url(redis['chartmuseum_db_index'], redis)
configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis)
configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis)

10
make/photon/prepare/utils/core.py
View File

@ -13,24 +13,16 @@ core_conf = os.path.join(config_dir, "core", "app.conf")
ca_download_dir = os.path.join(data_dir, 'ca_download')
def prepare_core(config_dict, with_notary, with_trivy, with_chartmuseum):
def prepare_core(config_dict, with_notary, with_trivy):
prepare_dir(ca_download_dir, uid=DEFAULT_UID, gid=DEFAULT_GID)
prepare_dir(core_config_dir)
# Render Core
# set cache for chart repo server
# default set 'memory' mode, if redis is configured then set to 'redis'
if len(config_dict['redis_url_core']) > 0:
chart_cache_driver = "redis"
else:
chart_cache_driver = "memory"
render_jinja(
core_env_template_path,
core_conf_env,
chart_cache_driver=chart_cache_driver,
with_notary=with_notary,
with_trivy=with_trivy,
with_chartmuseum=with_chartmuseum,
csrf_key=generate_random_string(32),
**config_dict)

4
make/photon/prepare/utils/docker_compose.py
View File

@ -8,7 +8,7 @@ docker_compose_template_path = os.path.join(templates_dir, 'docker_compose', 'do
docker_compose_yml_path = '/compose_location/docker-compose.yml'
# render docker-compose
def prepare_docker_compose(configs, with_trivy, with_notary, with_chartmuseum):
def prepare_docker_compose(configs, with_trivy, with_notary):
versions = parse_versions()
VERSION_TAG = versions.get('VERSION_TAG') or 'dev'
@ -18,7 +18,6 @@ def prepare_docker_compose(configs, with_trivy, with_notary, with_chartmuseum):
'redis_version': VERSION_TAG,
'notary_version': VERSION_TAG,
'trivy_adapter_version': VERSION_TAG,
'chartmuseum_version': VERSION_TAG,
'data_volume': configs['data_volume'],
'log_location': configs['log_location'],
'protocol': configs['protocol'],
@ -27,7 +26,6 @@ def prepare_docker_compose(configs, with_trivy, with_notary, with_chartmuseum):
'external_database': configs['external_database'],
'with_notary': with_notary,
'with_trivy': with_trivy,
'with_chartmuseum': with_chartmuseum
}
# if configs.get('registry_custom_ca_bundle_path'):

29
src/chartserver/base_test.go
View File

@ -1,29 +0,0 @@
package chartserver
import (
"net/http/httptest"
"net/url"
"github.com/goharbor/harbor/src/testing"
)
// createMockObjects create mock objects for chart repo related testing.
func createMockObjects() (*httptest.Server, *Controller, error) {
s := httptest.NewServer(testing.MockChartRepoHandler)
backendURL, err := url.Parse(s.URL)
if err != nil {
s.Close()
return nil, nil, err
}
mockController, err := NewController(backendURL)
if err != nil {
s.Close()
return nil, nil, err
}
return s, mockController, nil
}
// Http client
var httpClient = NewChartClient(nil)

229
src/chartserver/cache.go
View File

@ -1,229 +0,0 @@
package chartserver
import (
"context"
"encoding/json"
"errors"
"math"
"time"
beego_cache "github.com/beego/beego/v2/client/cache"
// Enable redis cache adaptor
_ "github.com/beego/beego/v2/client/cache/redis"
hlog "github.com/goharbor/harbor/src/lib/log"
)
const (
standardExpireTime = 3600 * time.Second
redisENVKey = "_REDIS_URL_CORE"
cacheDriverENVKey = "CHART_CACHE_DRIVER" // "memory" or "redis"
cacheDriverMem = "memory"
cacheDriverRedis = "redis"
cacheDriverRedisSentinel = "redis_sentinel"
cacheCollectionName = "helm_chart_cache"
maxTry = 10
)
// ChartCache is designed to cache some processed data for repeated accessing
// to improve the performance
type ChartCache struct {
// Cache driver
cache beego_cache.Cache
// Keep the driver type
driverType string
// To indicate if the chart cache is enabled
isEnabled bool
}
// ChartCacheConfig keeps the configurations of ChartCache
type ChartCacheConfig struct {
// Only support 'in-memory' and 'redis' now
DriverType string
// Align with config
Config string
}
// NewChartCache is constructor of ChartCache
// If return nil, that means no cache is enabled for chart repository server
func NewChartCache(config *ChartCacheConfig) *ChartCache {
// Never return nil object
chartCache := &ChartCache{
isEnabled: false,
}
// Double check the configurations are what we want
if config == nil {
return chartCache
}
if config.DriverType != cacheDriverMem && config.DriverType != cacheDriverRedis && config.DriverType != cacheDriverRedisSentinel {
return chartCache
}
if config.DriverType == cacheDriverRedis || config.DriverType == cacheDriverRedisSentinel {
if len(config.Config) == 0 {
return chartCache
}
}
// Try to create the upstream cache
cache := initCacheDriver(config)
if cache == nil {
return chartCache
}
// Cache enabled
chartCache.isEnabled = true
chartCache.driverType = config.DriverType
chartCache.cache = cache
return chartCache
}
// IsEnabled to indicate if the chart cache is successfully enabled
// The cache may be disabled if
//
// user does not set
// wrong configurations
func (chc *ChartCache) IsEnabled() bool {
return chc.isEnabled
}
// PutChart caches the detailed data of chart version
func (chc *ChartCache) PutChart(chart *ChartVersionDetails) {
ctx := context.Background()
// If cache is not enabled, do nothing
if !chc.IsEnabled() {
return
}
// As it's a valid json data anymore when retrieving back from redis cache,
// here we use separate methods to handle the data according to the driver type
if chart != nil {
var err error
switch chc.driverType {
case cacheDriverMem:
// Directly put object in
err = chc.cache.Put(ctx, chart.Metadata.Digest, chart, standardExpireTime)
case cacheDriverRedis, cacheDriverRedisSentinel:
// Marshal to json data before saving
var jsonData []byte
if jsonData, err = json.Marshal(chart); err == nil {
err = chc.cache.Put(ctx, chart.Metadata.Digest, jsonData, standardExpireTime)
}
default:
// Should not reach here, but still put guard code here
err = errors.New("meet invalid cache driver")
}
if err != nil {
// Just logged
hlog.Errorf("Failed to cache chart object with error: %s\n", err)
hlog.Warningf("If cache driver is using 'redis', please check the related configurations or the network connection")
}
}
}
// GetChart trys to retrieve it from the cache
// If hit, return the cached item;
// otherwise, nil object is returned
func (chc *ChartCache) GetChart(chartDigest string) *ChartVersionDetails {
// If cache is not enabled, do nothing
ctx := context.Background()
if !chc.IsEnabled() {
return nil
}
object, err := chc.cache.Get(ctx, chartDigest)
if err != nil {
hlog.Warningf("Failed to get cache value by key with error: %s", err)
}
if object != nil {
// Try to convert data
// First try the normal way
if chartDetails, ok := object.(*ChartVersionDetails); ok {
return chartDetails
}
// Maybe json bytes
if bytes, yes := object.([]byte); yes {
chartDetails := &ChartVersionDetails{}
err := json.Unmarshal(bytes, chartDetails)
if err == nil {
return chartDetails
}
// Just logged the error
hlog.Errorf("Failed to retrieve chart from cache with error: %s", err)
}
}
return nil
}
// Initialize the cache driver based on the config
func initCacheDriver(cacheConfig *ChartCacheConfig) beego_cache.Cache {
switch cacheConfig.DriverType {
case cacheDriverMem:
hlog.Info("Enable memory cache for chart caching")
return beego_cache.NewMemoryCache()
case cacheDriverRedis:
// New with retry
count := 0
for {
count++
redisCache, err := beego_cache.NewCache(cacheDriverRedis, cacheConfig.Config)
if err != nil {
// Just logged
hlog.Errorf("Failed to initialize redis cache: %s", err)
if count < maxTry {
<-time.After(time.Duration(backoff(count)) * time.Second)
continue
}
return nil
}
hlog.Info("Enable redis cache for chart caching")
return redisCache
}
case cacheDriverRedisSentinel:
// New with retry
count := 0
for {
count++
redisCache, err := beego_cache.NewCache(cacheDriverRedisSentinel, cacheConfig.Config)
if err != nil {
// Just logged
hlog.Errorf("Failed to initialize redis sentinel cache: %s", err)
if count < maxTry {
<-time.After(time.Duration(backoff(count)) * time.Second)
continue
}
return nil
}
hlog.Info("Enable redis sentinel cache for chart caching")
return redisCache
}
default:
break
}
// Any other cases
hlog.Info("No cache is enabled for chart caching")
return nil
}
// backoff: fast->slow->fast
func backoff(count int) int {
f := 5 - math.Abs((float64)(count)-5)
return (int)(math.Pow(2, f))
}

92
src/chartserver/cache_test.go
View File

@ -1,92 +0,0 @@
package chartserver
import (
"encoding/json"
"testing"
"helm.sh/helm/v3/pkg/chart"
helm_repo "helm.sh/helm/v3/pkg/repo"
)
var (
mockChart = &ChartVersionDetails{
Metadata: &helm_repo.ChartVersion{
Metadata: &chart.Metadata{
Name: "mock_chart",
Version: "0.1.0",
},
Digest: "mock_digest",
},
Dependencies: make([]*chart.Dependency, 0),
}
)
// Test the no cache set scenario
func TestNoCache(t *testing.T) {
chartCache := NewChartCache(nil)
if chartCache == nil {
t.Fatalf("cache instance should not be nil")
}
if chartCache.IsEnabled() {
t.Fatal("chart cache should not be enabled")
}
}
// Test the in memory cache
func TestInMemoryCache(t *testing.T) {
chartCache := NewChartCache(&ChartCacheConfig{
DriverType: cacheDriverMem,
})
if chartCache == nil {
t.Fatalf("cache instance should not be nil")
}
if !chartCache.IsEnabled() {
t.Fatal("chart cache should be enabled")
}
if chartCache.driverType != cacheDriverMem {
t.Fatalf("expect driver type %s but got %s", cacheDriverMem, chartCache.driverType)
}
chartCache.PutChart(mockChart)
theCachedChart := chartCache.GetChart(mockChart.Metadata.Digest)
if theCachedChart == nil || theCachedChart.Metadata.Name != mockChart.Metadata.Name {
t.Fatal("In memory cache does work")
}
}
// Test redis cache
// Failed to config redis cache and then use in memory instead
func TestRedisCache(t *testing.T) {
redisConfigV := make(map[string]string)
redisConfigV["key"] = cacheCollectionName
redisConfigV["conn"] = ":6379"
redisConfigV["dbNum"] = "0"
redisConfigV["password"] = ""
redisConfig, _ := json.Marshal(redisConfigV)
chartCache := NewChartCache(&ChartCacheConfig{
DriverType: cacheDriverRedis,
Config: string(redisConfig),
})
if chartCache == nil {
t.Fatalf("cache instance should not be nil")
}
if !chartCache.IsEnabled() {
t.Fatal("chart cache should be enabled")
}
if chartCache.driverType != cacheDriverRedis {
t.Fatalf("expect driver type '%s' but got '%s'", cacheDriverRedis, chartCache.driverType)
}
chartCache.PutChart(mockChart)
theCachedChart := chartCache.GetChart(mockChart.Metadata.Digest)
if theCachedChart == nil || theCachedChart.Metadata.Name != mockChart.Metadata.Name {
t.Fatal("In memory cache does work")
}
}

262
src/chartserver/chart_operator.go
View File

@ -1,262 +0,0 @@
package chartserver
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/Masterminds/semver"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
helm_repo "helm.sh/helm/v3/pkg/repo"
hlog "github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/label/model"
)
const (
readmeFileName = "README.md"
valuesFileName = "values.yaml"
)
// ChartVersion extends the helm ChartVersion with additional labels
type ChartVersion struct {
helm_repo.ChartVersion
Labels []*model.Label `json:"labels"`
}
// ChartVersions is an array of extended ChartVersion
type ChartVersions []*ChartVersion
// ChartVersionDetails keeps the detailed data info of the chart version
type ChartVersionDetails struct {
Metadata *helm_repo.ChartVersion `json:"metadata"`
Dependencies []*chart.Dependency `json:"dependencies"`
Values map[string]interface{} `json:"values"`
Files map[string]string `json:"files"`
Security *SecurityReport `json:"security"`
Labels []*model.Label `json:"labels"`
}
// SecurityReport keeps the info related with security
// e.g.: digital signature, vulnerability scanning etc.
type SecurityReport struct {
Signature *DigitalSignature `json:"signature"`
}
// DigitalSignature used to indicate if the chart has been signed
type DigitalSignature struct {
Signed bool `json:"signed"`
Provenance string `json:"prov_file"`
}
// ChartInfo keeps the information of the chart
type ChartInfo struct {
Name string `json:"name"`
TotalVersions uint32 `json:"total_versions"`
LatestVersion string `json:"latest_version"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
Icon string `json:"icon"`
Home string `json:"home"`
Deprecated bool `json:"deprecated"`
}
// ChartOperator is designed to process the contents of
// the specified chart version to get more details
type ChartOperator struct{}
// GetChartDetails parse the details from the provided content bytes
func (cho *ChartOperator) GetChartDetails(content []byte) (*ChartVersionDetails, error) {
chartData, err := cho.GetChartData(content)
if err != nil {
return nil, err
}
dependencies := chartData.Metadata.Dependencies
var values map[string]interface{}
var buf bytes.Buffer
files := make(map[string]string)
// Parse values
if chartData.Values != nil {
// values = parseRawValues([]byte(chartData.Values.GetRaw()))
if len(chartData.Values) > 0 {
c := chartutil.Values(chartData.Values)
ValYaml, err := c.YAML()
if err != nil {
return nil, err
}
err = c.Encode(&buf)
if err != nil {
return nil, err
}
values = parseRawValues(buf.Bytes())
// Append values.yaml file
files[valuesFileName] = ValYaml
}
}
// Append other files like 'README.md'
for _, v := range chartData.Files {
if v.Name == readmeFileName {
files[readmeFileName] = string(v.Data)
break
}
}
theChart := &ChartVersionDetails{
Dependencies: dependencies,
Values: values,
Files: files,
}
return theChart, nil
}
// GetChartList returns a reorganized chart list
func (cho *ChartOperator) GetChartList(content []byte) ([]*ChartInfo, error) {
if len(content) == 0 {
return nil, errors.New("zero content")
}
allCharts := make(map[string]helm_repo.ChartVersions)
if err := json.Unmarshal(content, &allCharts); err != nil {
return nil, err
}
chartList := make([]*ChartInfo, 0)
for key, chartVersions := range allCharts {
lVersion, oVersion := getTheTwoCharts(chartVersions)
if lVersion != nil && oVersion != nil {
chartInfo := &ChartInfo{
Name: key,
TotalVersions: uint32(len(chartVersions)),
}
chartInfo.Created = oVersion.Created
chartInfo.Home = lVersion.Home
chartInfo.Icon = lVersion.Icon
chartInfo.Deprecated = lVersion.Deprecated
chartInfo.LatestVersion = lVersion.Version
chartList = append(chartList, chartInfo)
}
}
// Sort the chart list by the updated time which is the create time
// of the latest version of the chart.
sort.Slice(chartList, func(i, j int) bool {
if chartList[i].Updated.Equal(chartList[j].Updated) {
return strings.Compare(chartList[i].Name, chartList[j].Name) < 0
}
return chartList[i].Updated.After(chartList[j].Updated)
})
return chartList, nil
}
// GetChartData returns raw data of chart
func (cho *ChartOperator) GetChartData(content []byte) (*chart.Chart, error) {
if len(content) == 0 {
return nil, errors.New("zero content")
}
reader := bytes.NewReader(content)
chartData, err := loader.LoadArchive(reader)
if err != nil {
return nil, err
}
return chartData, nil
}
// GetChartVersions returns the chart versions
func (cho *ChartOperator) GetChartVersions(content []byte) (ChartVersions, error) {
if len(content) == 0 {
return nil, errors.New("zero content")
}
chartVersions := make(ChartVersions, 0)
if err := json.Unmarshal(content, &chartVersions); err != nil {
return nil, err
}
return chartVersions, nil
}
// Get the latest and oldest chart versions
func getTheTwoCharts(chartVersions helm_repo.ChartVersions) (latestChart *helm_repo.ChartVersion, oldestChart *helm_repo.ChartVersion) {
if len(chartVersions) == 1 {
return chartVersions[0], chartVersions[0]
}
for _, chartVersion := range chartVersions {
currentV, err := semver.NewVersion(chartVersion.Version)
if err != nil {
// ignore it, just logged
hlog.Warningf("Malformed semversion %s for the chart %s", chartVersion.Version, chartVersion.Name)
continue
}
// Find latest chart
if latestChart == nil {
latestChart = chartVersion
} else {
lVersion, err := semver.NewVersion(latestChart.Version)
if err != nil {
// ignore it, just logged
hlog.Warningf("Malformed semversion %s for the chart %s", latestChart.Version, chartVersion.Name)
continue
}
if lVersion.LessThan(currentV) {
latestChart = chartVersion
}
}
if oldestChart == nil {
oldestChart = chartVersion
} else {
if oldestChart.Created.After(chartVersion.Created) {
oldestChart = chartVersion
}
}
}
return latestChart, oldestChart
}
// Parse the raw values to value map
func parseRawValues(rawValue []byte) map[string]interface{} {
valueMap := make(map[string]interface{})
if len(rawValue) == 0 {
return valueMap
}
values, err := chartutil.ReadValues(rawValue)
if err != nil || len(values) == 0 {
return valueMap
}
readValue(values, "", valueMap)
return valueMap
}
// Recursively read value
func readValue(values map[string]interface{}, keyPrefix string, valueMap map[string]interface{}) {
for key, value := range values {
longKey := key
if keyPrefix != "" {
longKey = fmt.Sprintf("%s.%s", keyPrefix, key)
}
if subValues, ok := value.(map[string]interface{}); ok {
readValue(subValues, longKey, valueMap)
} else {
valueMap[longKey] = value
}
}
}

48
src/chartserver/chart_operator_test.go
View File

@ -1,48 +0,0 @@
package chartserver
import (
"testing"
htesting "github.com/goharbor/harbor/src/testing"
)
func TestGetChartDetails(t *testing.T) {
chartOpr := ChartOperator{}
chartDetails, err := chartOpr.GetChartDetails(htesting.HelmChartContent)
if err != nil {
t.Fatal(err)
}
if len(chartDetails.Dependencies) == 0 {
t.Fatal("At least 1 dependency exitsing, but we got 0 now")
}
if len(chartDetails.Values) == 0 {
t.Fatal("At least 1 value existing, but we got 0 now")
}
if chartDetails.Values["adminserver.adminPassword"] != "Harbor12345" {
t.Fatalf("The value of 'adminserver.adminPassword' should be 'Harbor12345' but we got '%s' now", chartDetails.Values["adminserver.adminPassword"])
}
}
func TestGetChartList(t *testing.T) {
chartOpr := ChartOperator{}
infos, err := chartOpr.GetChartList(htesting.ChartListContent)
if err != nil {
t.Fatal(err)
}
if len(infos) != 2 {
t.Fatalf("Length of chart list should be 2, but we got %d now", len(infos))
}
firstInSortedList := infos[0]
if firstInSortedList.Name != "harbor" {
t.Fatalf("Expect the fist item of the sorted list to be 'harbor' but got '%s'", firstInSortedList.Name)
}
if firstInSortedList.LatestVersion != "0.2.0" {
t.Fatalf("Expect latest version '0.2.0' but got '%s'", firstInSortedList.LatestVersion)
}
}

148
src/chartserver/client.go
View File

@ -1,148 +0,0 @@
package chartserver
import (
"fmt"
"io"
"net/http"
"net/url"
"strings"
"sync"
"time"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/lib/errors"
tracelib "github.com/goharbor/harbor/src/lib/trace"
)
const (
clientTimeout = 30 * time.Second
maxIdleConnections = 10
idleConnectionTimeout = 30 * time.Second
)
var (
once sync.Once
chartTransport http.RoundTripper
)
// ChartClient is a http client to get the content from the external http server
type ChartClient struct {
// HTTP client
httpClient *http.Client
// Auth info
credential *Credential
}
// NewChartClient is constructor of ChartClient
// credential can be nil
func NewChartClient(credential *Credential) *ChartClient { // Create http client with customized timeouts
once.Do(func() {
chartTransport = commonhttp.NewTransport(
commonhttp.WithMaxIdleConns(maxIdleConnections),
commonhttp.WithIdleconnectionTimeout(idleConnectionTimeout),
)
if tracelib.Enabled() {
chartTransport = otelhttp.NewTransport(chartTransport)
}
})
client := &http.Client{
Timeout: clientTimeout,
Transport: chartTransport,
}
return &ChartClient{
httpClient: client,
credential: credential,
}
}
// GetContent get the bytes from the specified url
func (cc *ChartClient) GetContent(addr string) ([]byte, error) {
response, err := cc.sendRequest(addr, http.MethodGet, nil)
if err != nil {
err = errors.Wrap(err, "get content failed")
return nil, err
}
content, err := io.ReadAll(response.Body)
if err != nil {
err = errors.Wrap(err, "Read response body error")
return nil, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
text, err := extractError(content)
if err != nil {
err = errors.Wrap(err, "Extract content error failed")
return nil, err
}
return nil, &commonhttp.Error{
Code: response.StatusCode,
Message: text,
}
}
return content, nil
}
// DeleteContent sends deleting request to the addr to delete content
func (cc *ChartClient) DeleteContent(addr string) error {
response, err := cc.sendRequest(addr, http.MethodDelete, nil)
if err != nil {
return err
}
content, err := io.ReadAll(response.Body)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
text, err := extractError(content)
if err != nil {
return err
}
return &commonhttp.Error{
Code: response.StatusCode,
Message: text,
}
}
return nil
}
// sendRequest sends requests to the addr with the specified spec
func (cc *ChartClient) sendRequest(addr string, method string, body io.Reader) (*http.Response, error) {
if len(strings.TrimSpace(addr)) == 0 {
return nil, errors.New("empty url is not allowed")
}
fullURI, err := url.Parse(addr)
if err != nil {
err = errors.Wrap(err, "Invalid url")
return nil, err
}
request, err := http.NewRequest(method, addr, body)
if err != nil {
return nil, err
}
// Set basic auth
if cc.credential != nil {
request.SetBasicAuth(cc.credential.Username, cc.credential.Password)
}
response, err := cc.httpClient.Do(request)
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("send request %s %s failed", method, fullURI.Path))
return nil, err
}
return response, nil
}

84
src/chartserver/controller.go
View File

@ -1,84 +0,0 @@
package chartserver
import (
"errors"
"fmt"
"net/http"
"net/url"
"os"
hlog "github.com/goharbor/harbor/src/lib/log"
)
const (
userName = "chart_controller"
passwordKey = "CORE_SECRET"
)
// Credential keeps the username and password for the basic auth
type Credential struct {
Username string
Password string
}
// Controller is used to handle flows of related requests based on the corresponding handlers
// A reverse proxy will be created and managed to proxy the related traffics between API and
// backend chart server
type Controller struct {
// Proxy used to to transfer the traffic of requests
// It's mainly used to talk to the backend chart server
trafficProxy *ProxyEngine
// Parse and process the chart version to provide required info data
chartOperator *ChartOperator
// HTTP client used to call the realted APIs of the backend chart repositories
apiClient *ChartClient
// The access endpoint of the backend chart repository server
backendServerAddress *url.URL
// Cache the chart data
chartCache *ChartCache
}
// NewController is constructor of the chartserver.Controller
func NewController(backendServer *url.URL, middlewares ...func(http.Handler) http.Handler) (*Controller, error) {
if backendServer == nil {
return nil, errors.New("failed to create chartserver.Controller: backend sever address is required")
}
// Try to create credential
cred := &Credential{
Username: userName,
Password: os.Getenv(passwordKey),
}
// Creat cache
cacheCfg, err := getCacheConfig()
if err != nil {
// just log the error
// will not break the whole flow if failed to create cache
hlog.Errorf("failed to get cache configuration with error: %s", err)
}
cache := NewChartCache(cacheCfg)
if !cache.IsEnabled() {
hlog.Info("No cache is enabled for chart caching")
}
return &Controller{
backendServerAddress: backendServer,
// Use customized reverse proxy
trafficProxy: NewProxyEngine(backendServer, cred, middlewares...),
// Initialize chart operator for use
chartOperator: &ChartOperator{},
// Create http client with customized timeouts
apiClient: NewChartClient(cred),
chartCache: cache,
}, nil
}
// APIPrefix returns the API prefix path of calling backend chart service.
func (c *Controller) APIPrefix(namespace string) string {
return fmt.Sprintf("%s/api/%s/charts", c.backendServerAddress.String(), namespace)
}

21
src/chartserver/controller_test.go
View File

@ -1,21 +0,0 @@
package chartserver
import (
"fmt"
"testing"
)
// Test controller
func TestController(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
prefix := c.APIPrefix("fake")
expected := fmt.Sprintf("%s/api/%s/charts", s.URL, "fake")
if prefix != expected {
t.Fatalf("expect '%s' but got '%s'", expected, prefix)
}
}

97
src/chartserver/handler_interface.go
View File

@ -1,97 +0,0 @@
package chartserver
import (
"net/http"
"helm.sh/helm/v3/cmd/helm/search"
helm_repo "helm.sh/helm/v3/pkg/repo"
)
// ServiceHandler defines the related methods to handle kinds of chart service requests.
type ServiceHandler interface {
// ListCharts lists all the charts under the specified namespace.
//
// namespace string: the chart namespace.
//
// If succeed, a chart info list with nil error will be returned;
// otherwise, a non-nil error will be got.
ListCharts(namespace string) ([]*ChartInfo, error)
// Get all the chart versions of the specified chart under the namespace.
//
// namespace string: the chart namespace.
// chartName string: the name of the chart, e.g: "harbor"
//
// If succeed, a chart version list with nil error will be returned;
// otherwise, a non-nil error will be got.
GetChart(namespace, chartName string) (helm_repo.ChartVersions, error)
// Get the detailed info of the specified chart version under the namespace.
// The detailed info includes chart summary, dependencies, values and signature status etc.
//
// namespace string: the chart namespace.
// chartName string: the name of the chart, e.g: "harbor"
// version string: the SemVer version of the chart, e.g: "0.2.0"
//
// If succeed, chart version details with nil error will be returned;
// otherwise, a non-nil error will be got.
GetChartVersionDetails(namespace, chartName, version string) (*ChartVersionDetails, error)
// SearchChart search charts in the specified namespaces with the keyword q.
// RegExp mode is enabled as default.
// For each chart, only the latest version will shown in the result list if matched to avoid duplicated entries.
// Keep consistent with `helm search` command.
//
// q string : the searching keyword
// namespaces []string : the search namespace scope
//
// If succeed, a search result list with nil error will be returned;
// otherwise, a non-nil error will be got.
SearchChart(q string, namespaces []string) ([]*search.Result, error)
// GetIndexFile will read the index.yaml under all namespaces and merge them as a single one
// Please be aware that, to support this function, the backend chart repository server should
// enable multi-tenancies
//
// namespaces []string : all the namespaces with accessing permissions
//
// If succeed, a unified merged index file with nil error will be returned;
// otherwise, a non-nil error will be got.
GetIndexFile(namespaces []string) (*helm_repo.IndexFile, error)
// Get the chart summary of the specified chart version.
//
// namespace string: the chart namespace.
// chartName string: the name of the chart, e.g: "harbor"
// version string: the SemVer version of the chart, e.g: "0.2.0"
//
// If succeed, chart version summary with nil error will be returned;
// otherwise, a non-nil error will be got.
GetChartVersion(namespace, name, version string) (*helm_repo.ChartVersion, error)
// DeleteChart deletes all the chart versions of the specified chart under the namespace.
//
// namespace string: the chart namespace.
// chartName string: the name of the chart, e.g: "harbor"
//
// If succeed, a nil error will be returned;
// otherwise, a non-nil error will be got.
DeleteChart(namespace, chartName string) error
// GetCountOfCharts calculates and returns the total count of charts under the specified namespaces.
//
// namespaces []string : the namespaces to count charts
//
// If succeed, a unsigned integer with nil error will be returned;
// otherwise, a non-nil error will be got.
GetCountOfCharts(namespaces []string) (uint64, error)
}
// ProxyTrafficHandler defines the handler methods to handle the proxy traffic.
type ProxyTrafficHandler interface {
// Proxy the traffic to the backended server
//
// Req *http.Request : The incoming http request
// w http.ResponseWriter : The response writer reference
ProxyTraffic(w http.ResponseWriter, req *http.Request)
}

144
src/chartserver/handler_manipulation.go
View File

@ -1,144 +0,0 @@
package chartserver
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"strings"
"github.com/ghodss/yaml"
helm_repo "helm.sh/helm/v3/pkg/repo"
commonhttp "github.com/goharbor/harbor/src/common/http"
rep_event "github.com/goharbor/harbor/src/controller/event/handler/replication/event"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg/reg/model"
)
// ListCharts gets the chart list under the namespace
// See @ServiceHandler.ListCharts
func (c *Controller) ListCharts(namespace string) ([]*ChartInfo, error) {
if len(strings.TrimSpace(namespace)) == 0 {
return nil, errors.New("empty namespace when getting chart list")
}
content, err := c.apiClient.GetContent(c.APIPrefix(namespace))
if err != nil {
return nil, err
}
return c.chartOperator.GetChartList(content)
}
// GetChart returns all the chart versions under the specified chart
// See @ServiceHandler.GetChart
func (c *Controller) GetChart(namespace, chartName string) (ChartVersions, error) {
if len(namespace) == 0 {
return nil, errors.New("empty name when getting chart versions")
}
if len(chartName) == 0 {
return nil, errors.New("no chart name specified")
}
url := fmt.Sprintf("%s/%s", c.APIPrefix(namespace), chartName)
data, err := c.apiClient.GetContent(url)
if err != nil {
return nil, err
}
versions := make(ChartVersions, 0)
if err := json.Unmarshal(data, &versions); err != nil {
return nil, err
}
return versions, nil
}
// DeleteChartVersion will delete the specified version of the chart
// See @ServiceHandler.DeleteChartVersion
func (c *Controller) DeleteChartVersion(namespace, chartName, version string) error {
if len(namespace) == 0 {
return errors.New("empty namespace when deleting chart version")
}
if len(chartName) == 0 || len(version) == 0 {
return errors.New("invalid chart for deleting")
}
url := fmt.Sprintf("/api/chartrepo/%s/charts/%s/%s", namespace, chartName, version)
req, _ := http.NewRequest(http.MethodDelete, url, nil)
w := httptest.NewRecorder()
c.trafficProxy.ServeHTTP(w, req)
if w.Code != http.StatusOK {
text, err := extractError(w.Body.Bytes())
if err != nil {
return err
}
return &commonhttp.Error{
Code: w.Code,
Message: text,
}
}
// send notification to replication handler
// Todo: it used as the replacement of webhook, will be removed when webhook to be introduced.
if os.Getenv("UTTEST") != "true" {
go func() {
e := &rep_event.Event{
Type: rep_event.EventTypeChartDelete,
Resource: &model.Resource{
Type: model.ResourceTypeChart,
Deleted: true,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: fmt.Sprintf("%s/%s", namespace, chartName),
},
Artifacts: []*model.Artifact{
{
Tags: []string{version},
},
},
},
},
}
if err := rep_event.Handle(orm.Context(), e); err != nil {
log.Errorf("failed to handle event: %v", err)
}
}()
}
return nil
}
// GetChartVersion returns the summary of the specified chart version.
// See @ServiceHandler.GetChartVersion
func (c *Controller) GetChartVersion(namespace, name, version string) (*helm_repo.ChartVersion, error) {
if len(namespace) == 0 {
return nil, errors.New("empty namespace when getting summary of chart version")
}
if len(name) == 0 || len(version) == 0 {
return nil, errors.New("invalid chart when getting summary")
}
url := fmt.Sprintf("%s/%s/%s", c.APIPrefix(namespace), name, version)
content, err := c.apiClient.GetContent(url)
if err != nil {
return nil, err
}
chartVersion := &helm_repo.ChartVersion{}
if err := yaml.Unmarshal(content, chartVersion); err != nil {
return nil, err
}
return chartVersion, nil
}

88
src/chartserver/handler_manipulation_test.go
View File

@ -1,88 +0,0 @@
package chartserver
import (
"testing"
)
// Test get /api/:repo/charts/harbor
func TestGetChart(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
versions, err := c.GetChart("repo1", "harbor")
if err != nil {
t.Fatal(err)
}
if len(versions) != 2 {
t.Fatalf("expect 2 chart versions of harbor but got %d", len(versions))
}
}
// Test delete /api/:repo/charts/harbor/0.2.0
func TestDeleteChartVersion(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
if err := c.DeleteChartVersion("repo1", "harbor", "0.2.0"); err != nil {
t.Fatal(err)
}
}
// Test get /api/:repo/charts
func TestRetrieveChartList(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
chartList, err := c.ListCharts("repo1")
if err != nil {
t.Fatal(err)
}
if len(chartList) != 2 {
t.Fatalf("Expect to get 2 charts in the list but got %d", len(chartList))
}
foundItem := false
for _, chartInfo := range chartList {
if chartInfo.Name == "hello-helm" && chartInfo.TotalVersions == 2 {
foundItem = true
break
}
}
if !foundItem {
t.Fatalf("Expect chart 'hello-helm' with 2 versions but got nothing")
}
}
// Test the GetChartVersion in utility handler
func TestGetChartVersionSummary(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
chartV, err := c.GetChartVersion("repo1", "harbor", "0.2.0")
if err != nil {
t.Fatal(err)
}
if chartV.Name != "harbor" {
t.Fatalf("expect chart name 'harbor' but got '%s'", chartV.Name)
}
if chartV.Version != "0.2.0" {
t.Fatalf("expect chart version '0.2.0' but got '%s'", chartV.Version)
}
}

12
src/chartserver/handler_proxy_traffic.go
View File

@ -1,12 +0,0 @@
package chartserver
import (
"net/http"
)
// ProxyTraffic implements the interface method.
func (c *Controller) ProxyTraffic(w http.ResponseWriter, req *http.Request) {
if c.trafficProxy != nil {
c.trafficProxy.ServeHTTP(w, req)
}
}

138
src/chartserver/handler_proxy_traffic_test.go
View File

@ -1,138 +0,0 @@
package chartserver
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/ghodss/yaml"
helm_repo "helm.sh/helm/v3/pkg/repo"
htesting "github.com/goharbor/harbor/src/testing"
)
// The frontend server
var frontServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mockController.ProxyTraffic(w, r)
}))
var mockServer *httptest.Server
var mockController *Controller
// Prepare case
func TestStartMockServers(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
mockController = c
mockServer = s
frontServer.Start()
}
// Test /health
func TestGetHealthOfBaseHandler(t *testing.T) {
content, err := httpClient.GetContent(fmt.Sprintf("%s/api/chartrepo/health", frontServer.URL))
if err != nil {
t.Fatal(err)
}
status := make(map[string]interface{})
if err := json.Unmarshal(content, &status); err != nil {
t.Fatalf("Unmarshal error: %s, %s", err, content)
}
healthy, ok := status["health"].(bool)
if !ok || !healthy {
t.Fatalf("Expect healthy of server to be 'true' but got %v", status["health"])
}
}
// Get /repo1/index.yaml
func TestGetIndexYamlByRepo(t *testing.T) {
indexFile, err := getIndexYaml("/chartrepo/repo1/index.yaml")
if err != nil {
t.Fatal(err)
}
if len(indexFile.Entries) != 3 {
t.Fatalf("Expect index file with 3 entries, but got %d", len(indexFile.Entries))
}
}
// Test download /:repo/charts/chart.tar
// Use this case to test the proxy function
func TestDownloadChart(t *testing.T) {
content, err := httpClient.GetContent(fmt.Sprintf("%s/chartrepo/repo1/charts/harbor-0.2.0.tgz", frontServer.URL))
if err != nil {
t.Fatal(err)
}
gotSize := len(content)
expectSize := len(htesting.HelmChartContent)
if gotSize != expectSize {
t.Fatalf("Expect %d bytes data but got %d bytes", expectSize, gotSize)
}
}
// Get /api/repo1/charts/harbor
// 401 will be rewritten to 500 with specified error
func TestResponseRewrite(t *testing.T) {
response, err := http.Get(fmt.Sprintf("%s/chartrepo/repo3/charts/harbor-0.8.1.tgz", frontServer.URL))
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusInternalServerError {
t.Fatalf("Expect status code 500 but got %d", response.StatusCode)
}
bytes, err := io.ReadAll(response.Body)
if err != nil {
t.Fatalf("Read bytes from http response failed with error: %s", err)
}
defer response.Body.Close()
errObj := make(map[string]interface{})
if err = json.Unmarshal(bytes, &errObj); err != nil {
t.Fatalf("Unmarshal error: %s", err)
}
if msg, ok := errObj["error"]; !ok {
t.Fatal("Expect an error message from server but got nothing")
} else {
if !strings.Contains(msg.(string), "operation request from unauthorized source is rejected") {
t.Fatal("Missing the required error message")
}
}
}
// Clear env
func TestStopMockServers(t *testing.T) {
frontServer.Close()
mockServer.Close()
}
// Utility method for getting index yaml file
func getIndexYaml(path string) (*helm_repo.IndexFile, error) {
content, err := httpClient.GetContent(fmt.Sprintf("%s%s", frontServer.URL, path))
if err != nil {
return nil, err
}
indexFile := &helm_repo.IndexFile{}
if err := yaml.Unmarshal(content, indexFile); err != nil {
return nil, fmt.Errorf("unmarshal error: %s", err)
}
if indexFile == nil {
return nil, fmt.Errorf("got nil index yaml file")
}
return indexFile, nil
}

219
src/chartserver/handler_repo.go
View File

@ -1,219 +0,0 @@
package chartserver
import (
"fmt"
"path"
"strings"
"sync"
"time"
"github.com/ghodss/yaml"
helm_repo "helm.sh/helm/v3/pkg/repo"
hlog "github.com/goharbor/harbor/src/lib/log"
)
const (
maxWorkers = 10
// Keep consistent with 'helm search' command
searchMaxScore = 25
)
// Result returned by worker
type processedResult struct {
namespace string
indexFileOfRepo *helm_repo.IndexFile
}
// GetIndexFile will read the index.yaml under all namespaces and merge them as a single one
// Please be aware that, to support this function, the backend chart repository server should
// enable multi-tenancies
//
// See @ServiceHandler.GetIndexFile
func (c *Controller) GetIndexFile(namespaces []string) (*helm_repo.IndexFile, error) {
if len(namespaces) == 0 {
return emptyIndexFile(), nil
}
return c.getIndexYaml(namespaces)
}
// getIndexYaml will get the index yaml files for all the namespaces and merge them
// as one unified index yaml file.
func (c *Controller) getIndexYaml(namespaces []string) (*helm_repo.IndexFile, error) {
// The final merged index file
mergedIndexFile := &helm_repo.IndexFile{
APIVersion: "v1",
Entries: make(map[string]helm_repo.ChartVersions),
Generated: time.Now().Round(time.Second),
PublicKeys: []string{},
}
// Sync the output results from the retriever
resultChan := make(chan *processedResult, 1)
// Receive error
errorChan := make(chan error, 1)
// Signal chan for merging work
mergeDone := make(chan struct{}, 1)
// Total projects/namespaces
total := len(namespaces)
// Initialize
initialItemCount := maxWorkers
if total < maxWorkers {
initialItemCount = total
}
// Retrieve index.yaml for repositories
workerPool := make(chan struct{}, initialItemCount)
// Add initial tokens to the worker
for i := 0; i < initialItemCount; i++ {
workerPool <- struct{}{}
}
// Track all the background threads
waitGroup := new(sync.WaitGroup)
// Start the index files merging thread
go func() {
defer func() {
mergeDone <- struct{}{}
}()
for res := range resultChan {
c.mergeIndexFile(res.namespace, mergedIndexFile, res.indexFileOfRepo)
}
}()
// Retrieve the index files for the repositories
// and blocking here
var err error
LOOP:
for _, ns := range namespaces {
// Check if error has occurred in some goroutines
select {
case err = <-errorChan:
break LOOP
default:
// do nothing
}
// Apply one token before processing
<-workerPool
waitGroup.Add(1)
go func(ns string) {
defer func() {
waitGroup.Done() // done
// Return the worker back to the worker
workerPool <- struct{}{}
}()
indexFile, err := c.getIndexYamlWithNS(ns)
if err != nil {
if len(errorChan) == 0 {
// Only need one error as failure signal
errorChan <- err
}
return
}
// Output
resultChan <- &processedResult{
namespace: ns,
indexFileOfRepo: indexFile,
}
}(ns)
}
// Hold util all the retrieving work are done
waitGroup.Wait()
// close merge channel
close(resultChan)
// Wait until merging thread quit
<-mergeDone
// All the threads are done
// Make sure error in the chan is read
if err == nil && len(errorChan) > 0 {
err = <-errorChan
}
// Met an error
if err != nil {
return nil, err
}
// Remove duplicated keys in public key list
hash := make(map[string]string)
for _, key := range mergedIndexFile.PublicKeys {
hash[key] = key
}
mergedIndexFile.PublicKeys = []string{}
for k := range hash {
mergedIndexFile.PublicKeys = append(mergedIndexFile.PublicKeys, k)
}
return mergedIndexFile, nil
}
// Get the index yaml file under the specified namespace from the backend server
func (c *Controller) getIndexYamlWithNS(namespace string) (*helm_repo.IndexFile, error) {
// Join url path
url := path.Join(namespace, "index.yaml")
url = fmt.Sprintf("%s/%s", c.backendServerAddress.String(), url)
hlog.Debugf("Getting index.yaml from '%s'", url)
content, err := c.apiClient.GetContent(url)
if err != nil {
return nil, err
}
// Traverse to index file object for merging
indexFile := helm_repo.NewIndexFile()
if err := yaml.Unmarshal(content, indexFile); err != nil {
return nil, err
}
return indexFile, nil
}
// Merge the content of mergingIndexFile to the baseIndex
// The chart url should be without --chart-url prefix
func (c *Controller) mergeIndexFile(namespace string,
baseIndex *helm_repo.IndexFile,
mergingIndexFile *helm_repo.IndexFile) {
// Append entries
for chartName, chartVersions := range mergingIndexFile.Entries {
nameWithNS := fmt.Sprintf("%s/%s", namespace, chartName)
for _, version := range chartVersions {
version.Name = nameWithNS
// Currently there is only one url
for index, url := range version.URLs {
if !strings.HasPrefix(url, "http") {
version.URLs[index] = path.Join(namespace, url)
}
}
}
// Appended
baseIndex.Entries[nameWithNS] = chartVersions
}
// Update generated time
if mergingIndexFile.Generated.After(baseIndex.Generated) {
baseIndex.Generated = mergingIndexFile.Generated
}
// Merge public keys
baseIndex.PublicKeys = append(baseIndex.PublicKeys, mergingIndexFile.PublicKeys...)
}
// Generate empty index file
func emptyIndexFile() *helm_repo.IndexFile {
emptyIndexFile := &helm_repo.IndexFile{}
emptyIndexFile.Generated = time.Now().Round(time.Second)
return emptyIndexFile
}

32
src/chartserver/handler_repo_test.go
View File

@ -1,32 +0,0 @@
package chartserver
import "testing"
// Test get /index.yaml
func TestGetIndexFile(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
namespaces := []string{"repo1", "repo2"}
indexFile, err := c.GetIndexFile(namespaces)
if err != nil {
t.Fatal(err)
}
if len(indexFile.Entries) != 5 {
t.Fatalf("Expect index file with 5 entries, but got %d", len(indexFile.Entries))
}
_, ok := indexFile.Entries["repo1/harbor"]
if !ok {
t.Fatal("Expect chart entry 'repo1/harbor' but got nothing")
}
_, ok = indexFile.Entries["repo2/harbor"]
if !ok {
t.Fatal("Expect chart entry 'repo2/harbor' but got nothing")
}
}

233
src/chartserver/handler_utility.go
View File

@ -1,233 +0,0 @@
package chartserver
import (
"fmt"
"path"
"strings"
"sync"
"helm.sh/helm/v3/cmd/helm/search"
"github.com/goharbor/harbor/src/lib/config"
"github.com/goharbor/harbor/src/lib/errors"
hlog "github.com/goharbor/harbor/src/lib/log"
)
const (
maxDeletionThreads = 10
)
// GetCountOfCharts calculates and returns the total count of charts under the specified namespaces.
// See @ServiceHandler.GetCountOfCharts
func (c *Controller) GetCountOfCharts(namespaces []string) (uint64, error) {
if len(namespaces) == 0 {
return 0, nil // Directly return 0 instead of non-nil error
}
indexFile, err := c.getIndexYaml(namespaces)
if err != nil {
return 0, err
}
return (uint64)(len(indexFile.Entries)), nil
}
// DeleteChart deletes all the chart versions of the specified chart under the namespace.
// See @ServiceHandler.DeleteChart
func (c *Controller) DeleteChart(namespace, chartName string) error {
if len(strings.TrimSpace(namespace)) == 0 {
return errors.New("empty namespace when deleting chart")
}
if len(strings.TrimSpace(chartName)) == 0 {
return errors.New("empty chart name when deleting chart")
}
url := fmt.Sprintf("%s/%s", c.APIPrefix(namespace), chartName)
content, err := c.apiClient.GetContent(url)
if err != nil {
return err
}
allVersions, err := c.chartOperator.GetChartVersions(content)
if err != nil {
return err
}
// Let's delete the versions in parallel
// The number of goroutine is controlled by the const maxDeletionThreads
qSize := len(allVersions)
if qSize > maxDeletionThreads {
qSize = maxDeletionThreads
}
tokenQueue := make(chan struct{}, qSize)
errChan := make(chan error, 1)
waitGroup := new(sync.WaitGroup)
waitGroup.Add(len(allVersions))
// Append initial tokens
for i := 0; i < qSize; i++ {
tokenQueue <- struct{}{}
}
// Collect errors
errs := make([]error, 0)
errWrapper := make(chan error, 1)
go func() {
defer func() {
// pass to the out func
if len(errs) > 0 {
errWrapper <- fmt.Errorf("%v", errs)
}
close(errWrapper)
}()
for deletionErr := range errChan {
errs = append(errs, deletionErr)
}
}()
// Schedule deletion tasks
for _, deletingVersion := range allVersions {
// Apply for token first
// If no available token, pending here
<-tokenQueue
// Got one token
go func(deletingVersion *ChartVersion) {
defer func() {
// return the token back
tokenQueue <- struct{}{}
// done
waitGroup.Done()
}()
if err := c.DeleteChartVersion(namespace, chartName, deletingVersion.Version); err != nil {
errChan <- err
}
}(deletingVersion)
}
// Wait all goroutines are done
waitGroup.Wait()
// Safe to quit error collection goroutine
close(errChan)
err = <-errWrapper
return err
}
// GetChartVersionDetails get the specified version for one chart
// This handler should return the details of the chart version,
// maybe including metadata,dependencies and values etc.
// See @ServiceHandler.GetChartVersionDetails
func (c *Controller) GetChartVersionDetails(namespace, chartName, version string) (*ChartVersionDetails, error) {
chartV, err := c.GetChartVersion(namespace, chartName, version)
if err != nil {
return nil, err
}
// Query cache
chartDetails := c.chartCache.GetChart(chartV.Digest)
if chartDetails == nil {
// NOT hit!!
content, err := c.getChartVersionContent(namespace, chartV.URLs[0])
if err != nil {
return nil, err
}
// Process bytes and get more details of chart version
chartDetails, err = c.chartOperator.GetChartDetails(content)
if err != nil {
return nil, err
}
chartDetails.Metadata = chartV
// Put it into the cache for next access
c.chartCache.PutChart(chartDetails)
} else {
// Just logged
hlog.Debugf("Get detailed data from cache for chart: %s:%s (%s)",
chartDetails.Metadata.Name,
chartDetails.Metadata.Version,
chartDetails.Metadata.Digest)
}
// The change of prov file will not cause any influence to the digest of chart,
// and then the digital signature status should be not cached
//
// Generate the security report
// prov file share same endpoint with the chart version
// Just add .prov suffix to the chart version to form the path of prov file
// Anyway, there will be a report about the digital signature status
chartDetails.Security = &SecurityReport{
Signature: &DigitalSignature{
Signed: false,
},
}
// Try to get the prov file to confirm if it is exitsing
provFilePath := fmt.Sprintf("%s.prov", chartV.URLs[0])
provBytes, err := c.getChartVersionContent(namespace, provFilePath)
if err == nil && len(provBytes) > 0 {
chartDetails.Security.Signature.Signed = true
chartDetails.Security.Signature.Provenance = provFilePath
} else {
// Just log it
hlog.Debugf("Failed to get prov file for chart %s with error: %s, got %d bytes", chartV.Name, err.Error(), len(provBytes))
}
return chartDetails, nil
}
// SearchChart search charts in the specified namespaces with the keyword q.
// RegExp mode is enabled as default.
// For each chart, only the latest version will shown in the result list if matched to avoid duplicated entries.
// Keep consistent with `helm search` command.
func (c *Controller) SearchChart(q string, namespaces []string) ([]*search.Result, error) {
if len(q) == 0 || len(namespaces) == 0 {
// Return empty list
return []*search.Result{}, nil
}
// Get the merged index yaml file of the namespaces
ind, err := c.getIndexYaml(namespaces)
if err != nil {
return nil, err
}
// Build the search index
index := search.NewIndex()
// As the repo name is already merged into the index yaml, we use empty repo name.
// Set 'All' to false to return only one version for each chart.
index.AddRepo("", ind, false)
// Search
// RegExp is enabled
results, err := index.Search(q, searchMaxScore, true)
if err != nil {
return nil, err
}
// Sort results.
search.SortScore(results)
return results, nil
}
// Get the content bytes of the chart version
func (c *Controller) getChartVersionContent(namespace string, subPath string) ([]byte, error) {
var url string
if strings.HasPrefix(subPath, "http") {
extEndpoint, err := config.ExtEndpoint()
if err != nil {
return nil, errors.Wrap(err, "can not get ext endpoint")
}
url = strings.TrimPrefix(subPath, fmt.Sprintf("%s/%s", extEndpoint, "chartrepo/"))
} else {
url = path.Join(namespace, subPath)
}
url = fmt.Sprintf("%s/%s", c.backendServerAddress.String(), url)
return c.apiClient.GetContent(url)
}

114
src/chartserver/handler_utility_test.go
View File

@ -1,114 +0,0 @@
package chartserver
import (
"testing"
)
// Test the function GetCountOfCharts
func TestGetCountOfCharts(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
count, err := c.GetCountOfCharts([]string{})
if err != nil {
t.Fatalf("expect nil error but got %s", err)
}
if count != 0 {
t.Fatalf("expect 0 but got %d", count)
}
namespaces := []string{"repo1", "repo2"}
count, err = c.GetCountOfCharts(namespaces)
if err != nil {
t.Fatalf("expect nil error but got %s", err)
}
if count != 5 {
t.Fatalf("expect 5 but got %d", count)
}
_, err = c.GetCountOfCharts([]string{"not-existing-ns"})
if err == nil {
t.Fatal("expect non-nil error but got nil one")
}
}
// Test the function DeleteChart
func TestDeleteChart(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
if err := c.DeleteChart("repo1", "harbor"); err != nil {
t.Fatal(err)
}
}
// Test get /api/:repo/charts/:chart_name/:version
func TestGetChartVersion(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
chartVersion, err := c.GetChartVersionDetails("repo1", "harbor", "0.2.0")
if err != nil {
t.Fatal(err)
}
if chartVersion.Metadata.Name != "harbor" {
t.Fatalf("Expect harbor chart version but got %s", chartVersion.Metadata.Name)
}
if chartVersion.Metadata.Version != "0.2.0" {
t.Fatalf("Expect version '0.2.0' but got version %s", chartVersion.Metadata.Version)
}
if len(chartVersion.Dependencies) != 1 {
t.Fatalf("Expect 1 dependency but got %d ones", len(chartVersion.Dependencies))
}
if len(chartVersion.Values) != 99 {
t.Fatalf("Expect 99 k-v values but got %d", len(chartVersion.Values))
}
}
// Test get /api/:repo/charts/:chart_name/:version with none-existing version
func TestGetChartVersionWithError(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
_, err = c.GetChartVersionDetails("repo1", "harbor", "1.0.0")
if err == nil {
t.Fatal("Expect an error but got nil")
}
}
// Test the chart searching
func TestChartSearching(t *testing.T) {
s, c, err := createMockObjects()
if err != nil {
t.Fatal(err)
}
defer s.Close()
namespaces := []string{"repo1", "repo2"}
q := "harbor"
results, err := c.SearchChart(q, namespaces)
if err != nil {
t.Fatalf("expect nil error but got '%s'", err)
}
if len(results) != 2 {
t.Fatalf("expect 2 results but got %d", len(results))
}
}

254
src/chartserver/redis_sentinel.go
View File

@ -1,254 +0,0 @@
package chartserver
import (
"context"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/FZambia/sentinel"
"github.com/beego/beego/v2/client/cache"
"github.com/gomodule/redigo/redis"
)
var (
// DefaultKey the collection name of redis for cache adapter.
DefaultKey = "beecacheRedis"
)
// Cache is Redis cache adapter.
type Cache struct {
p *redis.Pool // redis connection pool
conninfo string
dbNum int
key string
password string
maxIdle int
masterName string
}
// NewRedisCache create new redis cache with default collection name.
func NewRedisCache() cache.Cache {
return &Cache{key: DefaultKey}
}
// actually do the redis cmds, args[0] must be the key name.
func (rc *Cache) do(commandName string, args ...interface{}) (reply interface{}, err error) {
if len(args) < 1 {
return nil, errors.New("missing required arguments")
}
args[0] = rc.associate(args[0])
c := rc.p.Get()
defer c.Close()
return c.Do(commandName, args...)
}
// associate with config key.
func (rc *Cache) associate(originKey interface{}) string {
return fmt.Sprintf("%s:%s", rc.key, originKey)
}
// Get cache from redis.
func (rc *Cache) Get(ctx context.Context, key string) (interface{}, error) {
v, err := rc.do("GET", key)
if err != nil {
return nil, err
}
return v, err
}
// GetMulti get cache from redis.
func (rc *Cache) GetMulti(ctx context.Context, keys []string) ([]interface{}, error) {
c := rc.p.Get()
defer c.Close()
var args []interface{}
for _, key := range keys {
args = append(args, rc.associate(key))
}
values, err := redis.Values(c.Do("MGET", args...))
if err != nil {
return nil, err
}
return values, nil
}
// Put put cache to redis.
func (rc *Cache) Put(ctx context.Context, key string, val interface{}, timeout time.Duration) error {
_, err := rc.do("SETEX", key, int64(timeout/time.Second), val)
return err
}
// Delete delete cache in redis.
func (rc *Cache) Delete(ctx context.Context, key string) error {
_, err := rc.do("DEL", key)
return err
}
// IsExist check cache's existence in redis.
func (rc *Cache) IsExist(ctx context.Context, key string) (bool, error) {
v, err := redis.Bool(rc.do("EXISTS", key))
if err != nil {
return false, err
}
return v, nil
}
// Incr increase counter in redis.
func (rc *Cache) Incr(ctx context.Context, key string) error {
_, err := redis.Bool(rc.do("INCRBY", key, 1))
return err
}
// Decr decrease counter in redis.
func (rc *Cache) Decr(ctx context.Context, key string) error {
_, err := redis.Bool(rc.do("INCRBY", key, -1))
return err
}
// ClearAll clean all cache in redis. delete this redis collection.
func (rc *Cache) ClearAll(ctx context.Context) error {
c := rc.p.Get()
defer c.Close()
cachedKeys, err := redis.Strings(c.Do("KEYS", rc.key+":*"))
if err != nil {
return err
}
for _, str := range cachedKeys {
if _, err = c.Do("DEL", str); err != nil {
return err
}
}
return err
}
// StartAndGC start redis cache adapter.
// config is like {"key":"collection key","conn":"connection info","dbNum":"0","masterName":"mymaster"}
// the cache item in redis are stored forever,
// so no gc operation.
func (rc *Cache) StartAndGC(config string) error {
var cf map[string]string
err := json.Unmarshal([]byte(config), &cf)
if err != nil {
return err
}
if _, ok := cf["key"]; !ok {
cf["key"] = DefaultKey
}
if _, ok := cf["masterName"]; !ok {
return errors.New("config has no masterName")
}
if _, ok := cf["conn"]; !ok {
return errors.New("config has no conn key")
}
// Format redis://<password>@<host>:<port>
cf["conn"] = strings.Replace(cf["conn"], "redis://", "", 1)
cf["conn"] = strings.Replace(cf["conn"], "redis_sentinel://", "", 1)
if i := strings.Index(cf["conn"], "@"); i > -1 {
cf["password"] = cf["conn"][0:i]
cf["conn"] = cf["conn"][i+1:]
}
if _, ok := cf["dbNum"]; !ok {
cf["dbNum"] = "0"
}
if _, ok := cf["password"]; !ok {
cf["password"] = ""
}
if _, ok := cf["maxIdle"]; !ok {
cf["maxIdle"] = "3"
}
rc.key = cf["key"]
rc.masterName = cf["masterName"]
rc.conninfo = cf["conn"]
rc.dbNum, _ = strconv.Atoi(cf["dbNum"])
rc.password = cf["password"]
rc.maxIdle, _ = strconv.Atoi(cf["maxIdle"])
rc.connectInit()
c := rc.p.Get()
defer c.Close()
return c.Err()
}
// connect to redis.
func (rc *Cache) connectInit() {
dialFunc := func() (c redis.Conn, err error) {
c, err = redis.Dial("tcp", rc.conninfo)
if err != nil {
return nil, err
}
if rc.password != "" {
if _, err := c.Do("AUTH", rc.password); err != nil {
c.Close()
return nil, err
}
}
_, selecterr := c.Do("SELECT", rc.dbNum)
if selecterr != nil {
c.Close()
return nil, selecterr
}
return
}
// initialize a new pool
rc.p = &redis.Pool{
MaxIdle: rc.maxIdle,
IdleTimeout: 180 * time.Second,
Dial: dialFunc,
}
var sentinelOptions []redis.DialOption
redisOptions := sentinelOptions
if rc.password != "" {
redisOptions = append(redisOptions, redis.DialPassword(rc.password))
}
redisOptions = append(redisOptions, redis.DialDatabase(rc.dbNum))
sntnl := &sentinel.Sentinel{
Addrs: strings.Split(rc.conninfo, ","),
MasterName: rc.masterName,
Dial: func(addr string) (redis.Conn, error) {
fmt.Println("chart dial redis sentinel:", addr)
c, err := redis.Dial("tcp", addr, sentinelOptions...)
if err != nil {
return nil, err
}
return c, nil
},
}
rc.p = &redis.Pool{
Dial: func() (redis.Conn, error) {
masterAddr, err := sntnl.MasterAddr()
if err != nil {
return nil, err
}
fmt.Println("chart dial redis master:", masterAddr, "db:", rc.dbNum)
return redis.Dial("tcp", masterAddr, redisOptions...)
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
if !sentinel.TestRole(c, "master") {
return errors.New("role check failed")
}
return nil
},
MaxIdle: rc.maxIdle,
IdleTimeout: 180 * time.Second,
}
}
func init() {
cache.Register("redis_sentinel", NewRedisCache)
}

241
src/chartserver/reverse_proxy.go
View File

@ -1,241 +0,0 @@
package chartserver
import (
"bytes"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/http/httputil"
"net/url"
"os"
"strconv"
"strings"
"time"
"github.com/goharbor/harbor/src/common"
commonhttp "github.com/goharbor/harbor/src/common/http"
rep_event "github.com/goharbor/harbor/src/controller/event/handler/replication/event"
"github.com/goharbor/harbor/src/controller/event/metadata"
hlog "github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
n_event "github.com/goharbor/harbor/src/pkg/notifier/event"
)
const (
agentHarbor = "HARBOR"
contentLengthHeader = "Content-Length"
defaultRepo = "library"
rootUploadingEndpoint = "/api/chartrepo/charts"
chartRepoHealthEndpoint = "/api/chartrepo/health"
)
// ProxyEngine is used to proxy the related traffics
type ProxyEngine struct {
// The backend target server the traffic will be forwarded to
// Just in case we'll use it
backend *url.URL
// Use go reverse proxy as engine
engine http.Handler
}
// NewProxyEngine is constructor of NewProxyEngine
func NewProxyEngine(target *url.URL, cred *Credential, middlewares ...func(http.Handler) http.Handler) *ProxyEngine {
var engine http.Handler
engine = &httputil.ReverseProxy{
ErrorLog: log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile),
Director: func(req *http.Request) {
director(target, cred, req)
},
ModifyResponse: modifyResponse,
Transport: commonhttp.GetHTTPTransport(),
}
if len(middlewares) > 0 {
hlog.Info("New chart server traffic proxy with middlewares")
for i := len(middlewares) - 1; i >= 0; i-- {
engine = middlewares[i](engine)
}
}
return &ProxyEngine{
backend: target,
engine: engine,
}
}
// ServeHTTP serves the incoming http requests
func (pe *ProxyEngine) ServeHTTP(w http.ResponseWriter, req *http.Request) {
pe.engine.ServeHTTP(w, req)
}
// Overwrite the http requests
func director(target *url.URL, cred *Credential, req *http.Request) {
// Closure
targetQuery := target.RawQuery
// Overwrite the request URL to the target path
req.URL.Scheme = target.Scheme
req.URL.Host = target.Host
rewriteURLPath(req)
req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path)
if targetQuery == "" || req.URL.RawQuery == "" {
req.URL.RawQuery = targetQuery + req.URL.RawQuery
} else {
req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
}
if _, ok := req.Header["User-Agent"]; !ok {
req.Header.Set("User-Agent", agentHarbor)
}
// Add authentication header if it is existing
if cred != nil {
req.SetBasicAuth(cred.Username, cred.Password)
}
}
// Modify the http response
func modifyResponse(res *http.Response) error {
// Upload chart success, then to the notification to replication handler
if res.StatusCode == http.StatusCreated {
// 201 and has chart_upload_event context
// means this response is for uploading chart success.
chartUploadEvent := res.Request.Context().Value(common.ChartUploadCtxKey)
e, ok := chartUploadEvent.(*rep_event.Event)
if !ok {
hlog.Error("failed to convert chart upload context into replication event.")
} else {
// Todo: it used as the replacement of webhook, will be removed when webhook to be introduced.
go func() {
if err := rep_event.Handle(orm.Context(), e); err != nil {
hlog.Errorf("failed to handle event: %v", err)
}
}()
// Trigger harbor webhook
if e != nil && e.Resource != nil && e.Resource.Metadata != nil && len(e.Resource.Metadata.Artifacts) > 0 &&
len(e.Resource.ExtendedInfo) > 0 {
event := &n_event.Event{}
metaData := &metadata.ChartUploadMetaData{
ChartMetaData: metadata.ChartMetaData{
ProjectName: e.Resource.ExtendedInfo["projectName"].(string),
ChartName: e.Resource.ExtendedInfo["chartName"].(string),
Versions: e.Resource.Metadata.Artifacts[0].Tags,
OccurAt: time.Now(),
Operator: e.Resource.ExtendedInfo["operator"].(string),
},
}
if err := event.Build(metaData); err == nil {
if err := event.Publish(); err != nil {
hlog.Errorf("failed to publish chart upload event: %v", err)
}
} else {
hlog.Errorf("failed to build chart upload event metadata: %v", err)
}
}
}
}
// Process downloading chart success webhook event
if res.StatusCode == http.StatusOK {
chartDownloadEvent := res.Request.Context().Value(common.ChartDownloadCtxKey)
eventMetaData, ok := chartDownloadEvent.(*metadata.ChartDownloadMetaData)
if ok && eventMetaData != nil {
// Trigger harbor webhook
event := &n_event.Event{}
if err := event.Build(eventMetaData); err == nil {
if err := event.Publish(); err != nil {
hlog.Errorf("failed to publish chart download event: %v", err)
}
} else {
hlog.Errorf("failed to build chart download event metadata: %v", err)
}
}
}
// Accept cases
// Success or redirect
if res.StatusCode >= http.StatusOK && res.StatusCode <= http.StatusTemporaryRedirect {
return nil
}
// Detect the 401 code, if it is,overwrite it to 500.
// We also re-write the error content to structural error object
errorObj := make(map[string]string)
if res.StatusCode == http.StatusUnauthorized {
errorObj["error"] = "operation request from unauthorized source is rejected"
res.StatusCode = http.StatusInternalServerError
} else {
// Extract the error and wrap it into the error object
data, err := io.ReadAll(res.Body)
if err != nil {
errorObj["error"] = fmt.Sprintf("%s: %s", res.Status, err.Error())
} else {
if err := json.Unmarshal(data, &errorObj); err != nil {
errorObj["error"] = string(data)
}
}
}
content, err := json.Marshal(errorObj)
if err != nil {
return err
}
size := len(content)
body := io.NopCloser(bytes.NewReader(content))
res.Body = body
res.ContentLength = int64(size)
res.Header.Set(contentLengthHeader, strconv.Itoa(size))
return nil
}
// Join the path
// Copy from the go reverse proxy
func singleJoiningSlash(a, b string) string {
aslash := strings.HasSuffix(a, "/")
bslash := strings.HasPrefix(b, "/")
switch {
case aslash && bslash:
return a + b[1:]
case !aslash && !bslash:
return a + "/" + b
}
return a + b
}
// Rewrite the incoming URL with the right backend URL pattern
// Remove 'chartrepo' from the endpoints of manipulation API
// Remove 'chartrepo' from the endpoints of repository services
func rewriteURLPath(req *http.Request) {
incomingURLPath := req.URL.Path
// Health check endpoint
if incomingURLPath == chartRepoHealthEndpoint {
req.URL.Path = "/health"
return
}
// Root uploading endpoint
if incomingURLPath == rootUploadingEndpoint {
req.URL.Path = strings.Replace(incomingURLPath, "chartrepo", defaultRepo, 1)
return
}
// Repository endpoints
if strings.HasPrefix(incomingURLPath, "/chartrepo") {
req.URL.Path = strings.TrimPrefix(incomingURLPath, "/chartrepo")
return
}
// API endpoints
if strings.HasPrefix(incomingURLPath, "/api/chartrepo") {
req.URL.Path = strings.Replace(incomingURLPath, "/chartrepo", "", 1)
return
}
}

55
src/chartserver/reverse_proxy_test.go
View File

@ -1,55 +0,0 @@
package chartserver
import (
"net/http"
"testing"
)
// Test the URL rewrite function
func TestURLRewrite(t *testing.T) {
req, err := createRequest(http.MethodGet, "/api/chartrepo/health")
if err != nil {
t.Fatal(err)
}
rewriteURLPath(req)
if req.URL.Path != "/health" {
t.Fatalf("Expect url format %s but got %s", "/health", req.URL.Path)
}
req, err = createRequest(http.MethodGet, "/api/chartrepo/library/charts")
if err != nil {
t.Fatal(err)
}
rewriteURLPath(req)
if req.URL.Path != "/api/library/charts" {
t.Fatalf("Expect url format %s but got %s", "/api/library/charts", req.URL.Path)
}
req, err = createRequest(http.MethodPost, "/api/chartrepo/charts")
if err != nil {
t.Fatal(err)
}
rewriteURLPath(req)
if req.URL.Path != "/api/library/charts" {
t.Fatalf("Expect url format %s but got %s", "/api/library/charts", req.URL.Path)
}
req, err = createRequest(http.MethodGet, "/chartrepo/library/index.yaml")
if err != nil {
t.Fatal(err)
}
rewriteURLPath(req)
if req.URL.Path != "/library/index.yaml" {
t.Fatalf("Expect url format %s but got %s", "/library/index.yaml", req.URL.Path)
}
}
func createRequest(method string, url string) (*http.Request, error) {
req, err := http.NewRequest(method, url, nil)
if err != nil {
return nil, err
}
req.RequestURI = url
return req, nil
}

141
src/chartserver/utils.go
View File

@ -1,141 +0,0 @@
package chartserver
import (
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"strconv"
"strings"
)
// Extract error object '{"error": "****---***"}' from the content if existing
// nil error will be returned if it does exist
func extractError(content []byte) (text string, err error) {
if len(content) == 0 {
return "", nil
}
errorObj := make(map[string]string)
err = json.Unmarshal(content, &errorObj)
if err != nil {
return "", err
}
if errText, ok := errorObj["error"]; ok {
return errText, nil
}
return "", nil
}
// Parse the redis configuration to the beego cache pattern
// redis://:password@host:6379/1
// redis+sentinel://anonymous:password@host1:26379,host2:26379/mymaster/1
func parseRedisConfig(redisConfigV string) (map[string]string, error) {
if len(redisConfigV) == 0 {
return nil, errors.New("empty redis config")
}
redisConfig := make(map[string]string)
redisConfig["key"] = cacheCollectionName
if !strings.Contains(redisConfigV, "//") {
redisConfigV = "redis://" + redisConfigV
}
u, err := url.Parse(redisConfigV)
if err != nil {
return nil, fmt.Errorf("bad _REDIS_URL:%s", redisConfigV)
}
if u.Scheme == "redis+sentinel" {
ps := strings.Split(u.Path, "/")
if len(ps) < 2 {
return nil, fmt.Errorf("bad redis sentinel url: no master name, %s", redisConfigV)
}
if _, err := strconv.Atoi(ps[1]); err == nil {
return nil, fmt.Errorf("bad redis sentinel url: master name should not be a number, %s", redisConfigV)
}
redisConfig["conn"] = u.Host
if u.User != nil {
password, isSet := u.User.Password()
if isSet {
redisConfig["password"] = password
}
}
if len(ps) > 2 {
if _, err := strconv.Atoi(ps[2]); err != nil {
return nil, fmt.Errorf("bad redis sentinel url: bad db, %s", redisConfigV)
}
redisConfig["dbNum"] = ps[2]
} else {
redisConfig["dbNum"] = "0"
}
redisConfig["masterName"] = ps[1]
} else if u.Scheme == "redis" {
redisConfig["conn"] = u.Host // host
if u.User != nil {
password, isSet := u.User.Password()
if isSet {
redisConfig["password"] = password
}
}
if len(u.Path) > 1 {
if _, err := strconv.Atoi(u.Path[1:]); err != nil {
return nil, fmt.Errorf("bad redis url: bad db, %s", redisConfigV)
}
redisConfig["dbNum"] = u.Path[1:]
} else {
redisConfig["dbNum"] = "0"
}
} else {
return nil, fmt.Errorf("bad redis scheme, %s", redisConfigV)
}
return redisConfig, nil
}
// What's the cache driver if it is set
func parseCacheDriver() (string, bool) {
driver, ok := os.LookupEnv(cacheDriverENVKey)
return strings.ToLower(driver), ok
}
// Get and parse the configuration for the chart cache
func getCacheConfig() (*ChartCacheConfig, error) {
driver, isSet := parseCacheDriver()
if !isSet {
return nil, nil
}
if driver != cacheDriverMem && driver != cacheDriverRedis {
return nil, fmt.Errorf("cache driver '%s' is not supported, only support 'memory' and 'redis'", driver)
}
if driver == cacheDriverMem {
return &ChartCacheConfig{
DriverType: driver,
}, nil
}
redisConfigV := os.Getenv(redisENVKey)
redisCfg, err := parseRedisConfig(redisConfigV)
if err != nil {
return nil, fmt.Errorf("failed to parse redis configurations from '%s' with error: %s", redisCfg, err)
}
if _, isSet := redisCfg["masterName"]; isSet {
driver = cacheDriverRedisSentinel
}
// Convert config map to string
cfgData, err := json.Marshal(redisCfg)
if err != nil {
return nil, fmt.Errorf("failed to parse redis configurations from '%s' with error: %s", redisCfg, err)
}
return &ChartCacheConfig{
DriverType: driver,
Config: string(cfgData),
}, nil
}

129
src/chartserver/utils_test.go
View File

@ -1,129 +0,0 @@
package chartserver
import (
"encoding/json"
"testing"
)
// Test the utility function parseRedisConfig
func TestParseRedisConfig(t *testing.T) {
// Case 1: empty addr
redisAddr := ""
if _, err := parseRedisConfig(redisAddr); err == nil {
t.Fatal("expect non nil error but got nil one if addr is empty")
}
// Case 2: short pattern, addr:port
redisAddr = "redis:6379"
if parsedConn, err := parseRedisConfig(redisAddr); err != nil {
t.Fatalf("expect nil error but got non nil one if addr is short pattern: %s\n", parsedConn)
}
// Case 3: long pattern but miss some parts
redisAddr = "redis:6379?idle_timeout_seconds=100"
if parsedConn, err := parseRedisConfig(redisAddr); err != nil {
t.Fatalf("expect nil error but got non nil one if addr is long pattern with some parts missing: %v\n", parsedConn)
} else {
if num, ok := parsedConn["dbNum"]; !ok || num != "0" {
t.Fatalf("expect 'dbNum:0' in the parsed conn str: %v\n", parsedConn)
}
}
// Case 4: long pattern
redisAddr = ":Passw0rd@redis:6379/1?idle_timeout_seconds=100"
if parsedConn, err := parseRedisConfig(redisAddr); err != nil {
t.Fatal("expect nil error but got non nil one if addr is long pattern")
} else {
if num, ok := parsedConn["dbNum"]; !ok || num != "1" {
t.Fatalf("expect 'dbNum:1' in the parsed conn str: %v", parsedConn)
}
if p, ok := parsedConn["password"]; !ok || p != "Passw0rd" {
t.Fatalf("expect 'password:Passw0rd' in the parsed conn str: %v", parsedConn)
}
}
// Case 5: sentinel but miss master name
redisAddr = "redis+sentinel://:Passw0rd@redis1:26379,redis2:26379/1?idle_timeout_seconds=100"
if _, err := parseRedisConfig(redisAddr); err == nil {
t.Fatal("expect no master name error but got nil")
}
// Case 6: sentinel
redisAddr = "redis+sentinel://:Passw0rd@redis1:26379,redis2:26379/mymaster/1?idle_timeout_seconds=100"
if parsedConn, err := parseRedisConfig(redisAddr); err != nil {
t.Fatal("expect nil error but got non nil one if addr is long pattern")
} else {
if num, ok := parsedConn["dbNum"]; !ok || num != "1" {
t.Fatalf("expect 'dbNum:0' in the parsed conn str: %v", parsedConn)
}
if p, ok := parsedConn["password"]; !ok || p != "Passw0rd" {
t.Fatalf("expect 'password:Passw0rd' in the parsed conn str: %v", parsedConn)
}
if v, ok := parsedConn["masterName"]; !ok || v != "mymaster" {
t.Fatalf("expect 'masterName:mymaster' in the parsed conn str: %v", parsedConn)
}
if v, ok := parsedConn["conn"]; !ok || v != "redis1:26379,redis2:26379" {
t.Fatalf("expect 'conn:redis1:26379,redis2:26379' in the parsed conn str: %v", parsedConn)
}
}
}
func TestGetCacheConfig(t *testing.T) {
t.Run("no cache set", func(t *testing.T) {
cacheConf, err := getCacheConfig()
if err != nil || cacheConf != nil {
t.Fatal("expect nil cache config and nil error but got non-nil one when parsing empty cache settings")
}
})
t.Run("unknown cache type", func(t *testing.T) {
t.Setenv(cacheDriverENVKey, "unknown")
_, err := getCacheConfig()
if err == nil {
t.Fatal("expect non-nil error but got nil one when parsing unknown cache type")
}
})
t.Run("in memory cache type", func(t *testing.T) {
t.Setenv(cacheDriverENVKey, cacheDriverMem)
memCacheConf, err := getCacheConfig()
if err != nil || memCacheConf == nil || memCacheConf.DriverType != cacheDriverMem {
t.Fatal("expect in memory cache driver but got invalid one")
}
})
t.Run("wrong redis cache conf", func(t *testing.T) {
t.Setenv(cacheDriverENVKey, cacheDriverRedis)
t.Setenv(redisENVKey, "")
_, err := getCacheConfig()
if err == nil {
t.Fatal("expect non-nil error but got nil one when parsing a invalid redis cache conf")
}
})
t.Run("redis cache conf", func(t *testing.T) {
t.Setenv(cacheDriverENVKey, cacheDriverRedis)
t.Setenv(redisENVKey, ":Passw0rd@redis:6379/1?idle_timeout_seconds=100")
redisConf, err := getCacheConfig()
if err != nil {
t.Fatalf("expect nil error but got non-nil one when parsing valid redis conf")
}
if redisConf == nil || redisConf.DriverType != cacheDriverRedis {
t.Fatal("expect redis cache driver but got invalid one")
}
conf := make(map[string]string)
if err = json.Unmarshal([]byte(redisConf.Config), &conf); err != nil {
t.Fatal(err)
}
if v, ok := conf["conn"]; !ok {
t.Fatal("expect 'conn' filed in the parsed conf but got nothing")
} else {
if v != "redis:6379" {
t.Fatalf("expect %s but got %s", "redis:6379", v)
}
}
})
}

7
src/common/const.go
View File

@ -46,7 +46,6 @@ const (
ResourceTypeProject = "p"
ResourceTypeRepository = "r"
ResourceTypeImage = "i"
ResourceTypeChart = "c"
ExtEndpoint = "ext_endpoint"
AUTHMode = "auth_mode"
@ -137,9 +136,6 @@ const (
LDAPGroupAdminDn = "ldap_group_admin_dn"
LDAPGroupMembershipAttribute = "ldap_group_membership_attribute"
DefaultRegistryControllerEndpoint = "http://registryctl:8080"
WithChartMuseum = "with_chartmuseum"
ChartRepoURL = "chart_repository_url"
DefaultChartRepoURL = "http://chartmuseum:9999"
DefaultPortalURL = "http://portal:8080"
DefaultRegistryCtlURL = "http://registryctl:8080"
// Use this prefix to distinguish harbor user, the prefix contains a special character($), so it cannot be registered as a harbor user.
@ -156,9 +152,6 @@ const (
AuthProxyRediretPath = "/c/authproxy/redirect"
ChartUploadCtxKey = contextKey("chart_upload_event")
ChartDownloadCtxKey = contextKey("chart_download_event")
// Global notification enable configuration
NotificationEnable = "notification_enable"

49
src/common/rbac/const.go
View File

@ -35,32 +35,29 @@ const (
// const resource variables
const (
ResourceAll = Resource("*") // resource match any other resources
ResourceConfiguration = Resource("configuration") // project configuration compatible for portal only
ResourceHelmChart = Resource("helm-chart")
ResourceHelmChartVersion = Resource("helm-chart-version")
ResourceHelmChartVersionLabel = Resource("helm-chart-version-label")
ResourceLabel = Resource("label")
ResourceLog = Resource("log")
ResourceLdapUser = Resource("ldap-user")
ResourceMember = Resource("member")
ResourceMetadata = Resource("metadata")
ResourceQuota = Resource("quota")
ResourceRepository = Resource("repository")
ResourceTagRetention = Resource("tag-retention")
ResourceImmutableTag = Resource("immutable-tag")
ResourceRobot = Resource("robot")
ResourceNotificationPolicy = Resource("notification-policy")
ResourceScan = Resource("scan")
ResourceScanner = Resource("scanner")
ResourceArtifact = Resource("artifact")
ResourceTag = Resource("tag")
ResourceAccessory = Resource("accessory")
ResourceArtifactAddition = Resource("artifact-addition")
ResourceArtifactLabel = Resource("artifact-label")
ResourcePreatPolicy = Resource("preheat-policy")
ResourcePreatInstance = Resource("preheat-instance")
ResourceSelf = Resource("") // subresource for self
ResourceAll = Resource("*") // resource match any other resources
ResourceConfiguration = Resource("configuration") // project configuration compatible for portal only
ResourceLabel = Resource("label")
ResourceLog = Resource("log")
ResourceLdapUser = Resource("ldap-user")
ResourceMember = Resource("member")
ResourceMetadata = Resource("metadata")
ResourceQuota = Resource("quota")
ResourceRepository = Resource("repository")
ResourceTagRetention = Resource("tag-retention")
ResourceImmutableTag = Resource("immutable-tag")
ResourceRobot = Resource("robot")
ResourceNotificationPolicy = Resource("notification-policy")
ResourceScan = Resource("scan")
ResourceScanner = Resource("scanner")
ResourceArtifact = Resource("artifact")
ResourceTag = Resource("tag")
ResourceAccessory = Resource("accessory")
ResourceArtifactAddition = Resource("artifact-addition")
ResourceArtifactLabel = Resource("artifact-label")
ResourcePreatPolicy = Resource("preheat-policy")
ResourcePreatInstance = Resource("preheat-instance")
ResourceSelf = Resource("") // subresource for self
ResourceAuditLog = Resource("audit-log")
ResourceCatalog = Resource("catalog")

52
src/common/rbac/project/rbac_role.go
View File

@ -68,20 +68,6 @@ var (
{Resource: rbac.ResourceImmutableTag, Action: rbac.ActionDelete},
{Resource: rbac.ResourceImmutableTag, Action: rbac.ActionList},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionCreate}, // upload helm chart
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionRead}, // download helm chart
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionDelete},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionList},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionCreate}, // upload helm chart version
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionRead}, // read and download helm chart version
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionDelete},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionList},
{Resource: rbac.ResourceHelmChartVersionLabel, Action: rbac.ActionCreate},
{Resource: rbac.ResourceHelmChartVersionLabel, Action: rbac.ActionDelete},
{Resource: rbac.ResourceHelmChartVersionLabel, Action: rbac.ActionList},
{Resource: rbac.ResourceConfiguration, Action: rbac.ActionRead},
{Resource: rbac.ResourceConfiguration, Action: rbac.ActionUpdate},
@ -173,20 +159,6 @@ var (
{Resource: rbac.ResourceImmutableTag, Action: rbac.ActionDelete},
{Resource: rbac.ResourceImmutableTag, Action: rbac.ActionList},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionCreate},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionRead},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionDelete},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionList},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionCreate},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionRead},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionDelete},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionList},
{Resource: rbac.ResourceHelmChartVersionLabel, Action: rbac.ActionCreate},
{Resource: rbac.ResourceHelmChartVersionLabel, Action: rbac.ActionDelete},
{Resource: rbac.ResourceHelmChartVersionLabel, Action: rbac.ActionList},
{Resource: rbac.ResourceConfiguration, Action: rbac.ActionRead},
{Resource: rbac.ResourceRobot, Action: rbac.ActionRead},
@ -245,18 +217,6 @@ var (
{Resource: rbac.ResourceTagRetention, Action: rbac.ActionList},
{Resource: rbac.ResourceTagRetention, Action: rbac.ActionOperate},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionCreate},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionRead},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionList},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionCreate},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionRead},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionList},
{Resource: rbac.ResourceHelmChartVersionLabel, Action: rbac.ActionCreate},
{Resource: rbac.ResourceHelmChartVersionLabel, Action: rbac.ActionDelete},
{Resource: rbac.ResourceHelmChartVersionLabel, Action: rbac.ActionList},
{Resource: rbac.ResourceConfiguration, Action: rbac.ActionRead},
{Resource: rbac.ResourceRobot, Action: rbac.ActionRead},
@ -301,12 +261,6 @@ var (
{Resource: rbac.ResourceRepository, Action: rbac.ActionList},
{Resource: rbac.ResourceRepository, Action: rbac.ActionPull},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionRead},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionList},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionRead},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionList},
{Resource: rbac.ResourceConfiguration, Action: rbac.ActionRead},
{Resource: rbac.ResourceRobot, Action: rbac.ActionRead},
@ -332,12 +286,6 @@ var (
{Resource: rbac.ResourceRepository, Action: rbac.ActionList},
{Resource: rbac.ResourceRepository, Action: rbac.ActionPull},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionRead},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionList},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionRead},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionList},
{Resource: rbac.ResourceConfiguration, Action: rbac.ActionRead},
{Resource: rbac.ResourceScan, Action: rbac.ActionRead},

6
src/common/rbac/project/rbac_util.go
View File

@ -30,12 +30,6 @@ var (
{Resource: rbac.ResourceRepository, Action: rbac.ActionList},
{Resource: rbac.ResourceRepository, Action: rbac.ActionPull},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionRead},
{Resource: rbac.ResourceHelmChart, Action: rbac.ActionList},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionRead},
{Resource: rbac.ResourceHelmChartVersion, Action: rbac.ActionList},
{Resource: rbac.ResourceScan, Action: rbac.ActionRead},
{Resource: rbac.ResourceScanner, Action: rbac.ActionRead},

1
src/common/utils/test/test.go
View File

@ -122,7 +122,6 @@ func GetUnitTestConfig() map[string]interface{} {
common.LDAPGroupSearchScope: 2,
common.LDAPGroupAdminDn: "cn=harbor_users,ou=groups,dc=example,dc=com",
common.WithNotary: "false",
common.WithChartMuseum: "false",
common.SelfRegistration: "true",
common.WithTrivy: "true",
common.TokenServiceURL: "http://core:8080/service/token",

2
src/controller/artifact/processor/processor.go
View File

@ -44,7 +44,7 @@ type Processor interface {
AbstractMetadata(ctx context.Context, artifact *artifact.Artifact, manifest []byte) error
// AbstractAddition abstracts the addition of the artifact.
// The additions are different for different artifacts:
// build history for image; values.yaml, readme and dependencies for chart, etc
// build history for image;
AbstractAddition(ctx context.Context, artifact *artifact.Artifact, additionType string) (addition *Addition, err error)
}

4
src/controller/event/handler/init.go
View File

@ -9,7 +9,6 @@ import (
"github.com/goharbor/harbor/src/controller/event/handler/p2p"
"github.com/goharbor/harbor/src/controller/event/handler/replication"
"github.com/goharbor/harbor/src/controller/event/handler/webhook/artifact"
"github.com/goharbor/harbor/src/controller/event/handler/webhook/chart"
"github.com/goharbor/harbor/src/controller/event/handler/webhook/quota"
"github.com/goharbor/harbor/src/controller/event/handler/webhook/scan"
"github.com/goharbor/harbor/src/controller/event/metadata"
@ -24,9 +23,6 @@ func init() {
_ = notifier.Subscribe(event.TopicPushArtifact, &artifact.Handler{})
_ = notifier.Subscribe(event.TopicPullArtifact, &artifact.Handler{})
_ = notifier.Subscribe(event.TopicDeleteArtifact, &artifact.Handler{})
_ = notifier.Subscribe(event.TopicUploadChart, &chart.Handler{})
_ = notifier.Subscribe(event.TopicDeleteChart, &chart.Handler{})
_ = notifier.Subscribe(event.TopicDownloadChart, &chart.Handler{})
_ = notifier.Subscribe(event.TopicQuotaExceed, &quota.Handler{})
_ = notifier.Subscribe(event.TopicQuotaWarning, &quota.Handler{})
_ = notifier.Subscribe(event.TopicScanningFailed, &scan.Handler{})

4
src/controller/event/handler/replication/event/event.go
View File

@ -21,11 +21,9 @@ const (
EventTypeArtifactPush = "artifact_push"
EventTypeArtifactDelete = "artifact_delete"
EventTypeTagDelete = "tag_delete"
EventTypeChartUpload = "chart_upload"
EventTypeChartDelete = "chart_delete"
)
// Event is the model that defines the image/chart pull/push event
// Event is the model that defines the image pull/push event
type Event struct {
Type string
Resource *model.Resource

3
src/controller/event/handler/replication/event/handler.go
View File

@ -37,8 +37,7 @@ func Handle(ctx context.Context, event *Event) error {
var policies []*repctlmodel.Policy
var err error
switch event.Type {
case EventTypeArtifactPush, EventTypeChartUpload, EventTypeTagDelete,
EventTypeArtifactDelete, EventTypeChartDelete:
case EventTypeArtifactPush, EventTypeTagDelete, EventTypeArtifactDelete:
policies, err = getRelatedPolicies(ctx, event.Resource)
default:
return fmt.Errorf("unsupported event type %s", event.Type)

122
src/controller/event/handler/webhook/chart/chart.go
View File

@ -1,122 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package chart
import (
"context"
"errors"
"fmt"
"github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/controller/event/handler/util"
"github.com/goharbor/harbor/src/controller/project"
"github.com/goharbor/harbor/src/lib/config"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/pkg/notifier/model"
proModels "github.com/goharbor/harbor/src/pkg/project/models"
)
// Handler preprocess chart event data
type Handler struct {
}
// Name ...
func (cph *Handler) Name() string {
return "ChartWebhook"
}
// Handle preprocess chart event data and then publish hook event
func (cph *Handler) Handle(ctx context.Context, value interface{}) error {
chartEvent, ok := value.(*event.ChartEvent)
if !ok {
return errors.New("invalid chart event type")
}
if chartEvent == nil || len(chartEvent.Versions) == 0 || len(chartEvent.ProjectName) == 0 || len(chartEvent.ChartName) == 0 {
return fmt.Errorf("data miss in chart event: %v", chartEvent)
}
prj, err := project.Ctl.Get(ctx, chartEvent.ProjectName, project.Metadata(true))
if err != nil {
log.Errorf("failed to find project[%s] for chart event: %v", chartEvent.ProjectName, err)
return err
}
policies, err := notification.PolicyMgr.GetRelatedPolices(ctx, prj.ProjectID, chartEvent.EventType)
if err != nil {
log.Errorf("failed to find policy for %s event: %v", chartEvent.EventType, err)
return err
}
// if cannot find policy including event type in project, return directly
if len(policies) == 0 {
log.Debugf("cannot find policy for %s event: %v", chartEvent.EventType, chartEvent)
return nil
}
payload, err := constructChartPayload(chartEvent, prj)
if err != nil {
return err
}
err = util.SendHookWithPolicies(policies, payload, chartEvent.EventType)
if err != nil {
return err
}
return nil
}
// IsStateful ...
func (cph *Handler) IsStateful() bool {
return false
}
func constructChartPayload(event *event.ChartEvent, project *proModels.Project) (*model.Payload, error) {
repoType := proModels.ProjectPrivate
if project.IsPublic() {
repoType = proModels.ProjectPublic
}
payload := &model.Payload{
Type: event.EventType,
OccurAt: event.OccurAt.Unix(),
EventData: &model.EventData{
Repository: &model.Repository{
Name: event.ChartName,
Namespace: event.ProjectName,
RepoFullName: fmt.Sprintf("%s/%s", event.ProjectName, event.ChartName),
RepoType: repoType,
},
},
Operator: event.Operator,
}
extURL, err := config.ExtEndpoint()
if err != nil {
return nil, fmt.Errorf("get external endpoint failed: %v", err)
}
resourcePrefix := fmt.Sprintf("%s/chartrepo/%s/charts/%s", extURL, event.ProjectName, event.ChartName)
for _, v := range event.Versions {
resURL := fmt.Sprintf("%s-%s.tgz", resourcePrefix, v)
resource := &model.Resource{
Tag: v,
ResourceURL: resURL,
}
payload.EventData.Resources = append(payload.EventData.Resources, resource)
}
return payload, nil
}

183
src/controller/event/handler/webhook/chart/chart_test.go
View File

@ -1,183 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package chart
import (
"context"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
testutils "github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/controller/project"
"github.com/goharbor/harbor/src/lib/config"
_ "github.com/goharbor/harbor/src/pkg/config/db"
_ "github.com/goharbor/harbor/src/pkg/config/inmemory"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/pkg/notification/policy/model"
proModels "github.com/goharbor/harbor/src/pkg/project/models"
projecttesting "github.com/goharbor/harbor/src/testing/controller/project"
"github.com/goharbor/harbor/src/testing/mock"
testingnotification "github.com/goharbor/harbor/src/testing/pkg/notification/policy"
)
func TestMain(m *testing.M) {
// do some initialization
testutils.InitDatabaseFromEnv()
os.Exit(m.Run())
}
func TestChartPreprocessHandler_Handle(t *testing.T) {
PolicyMgr := notification.PolicyMgr
defer func() {
notification.PolicyMgr = PolicyMgr
}()
policyMgrMock := &testingnotification.Manager{}
notification.PolicyMgr = policyMgrMock
ProjectCtl := project.Ctl
defer func() {
project.Ctl = ProjectCtl
}()
projectCtl := &projecttesting.Controller{}
project.Ctl = projectCtl
name := "project_for_test_chart_event_preprocess"
mock.OnAnything(projectCtl, "Get").Return(func(ctx context.Context, projectIDOrName interface{}, options ...project.Option) *proModels.Project {
return &proModels.Project{
Name: name,
OwnerID: 1,
Metadata: map[string]string{
proModels.ProMetaEnableContentTrust: "true",
proModels.ProMetaPreventVul: "true",
proModels.ProMetaSeverity: "Low",
proModels.ProMetaReuseSysCVEAllowlist: "false",
},
}
}, nil)
projectCtl.On("Get")
policyMgrMock.On("GetRelatedPolices", mock.Anything, mock.Anything, mock.Anything).Return([]*model.Policy{
{
ID: 1,
EventTypes: []string{
event.TopicUploadChart,
event.TopicDownloadChart,
event.TopicDeleteChart,
event.TopicPushArtifact,
event.TopicPullArtifact,
event.TopicDeleteArtifact,
event.TopicScanningFailed,
event.TopicScanningCompleted,
event.TopicQuotaExceed,
},
Targets: []model.EventTarget{
{
Type: "http",
Address: "http://127.0.0.1:8080",
},
},
},
}, nil)
handler := &Handler{}
config.Init()
type args struct {
data interface{}
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "Handler Want Error 1",
args: args{
data: nil,
},
wantErr: true,
},
{
name: "Handler Want Error 2",
args: args{
data: &event.ChartEvent{},
},
wantErr: true,
},
{
name: "Handler Want Error 3",
args: args{
data: &event.ChartEvent{
Versions: []string{
"v1.2.1",
},
ProjectName: "project_for_test_chart_event_preprocess",
},
},
wantErr: true,
},
{
name: "Handler Want Error 4",
args: args{
data: &event.ChartEvent{
Versions: []string{
"v1.2.1",
},
ProjectName: "project_for_test_chart_event_preprocess_not_exists",
ChartName: "testChart",
},
},
wantErr: true,
},
{
name: "Handler Want Error 5",
args: args{
data: &event.ChartEvent{
Versions: []string{
"v1.2.1",
},
ProjectName: "project_for_test_chart_event_preprocess",
ChartName: "testChart",
EventType: "uploadChart",
},
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := handler.Handle(context.TODO(), tt.args.data)
if tt.wantErr {
require.NotNil(t, err, "Error: %s", err)
return
}
assert.Nil(t, err)
})
}
}
func TestChartPreprocessHandler_IsStateful(t *testing.T) {
handler := &Handler{}
assert.False(t, handler.IsStateful())
}
func TestChartPreprocessHandler_Name(t *testing.T) {
handler := &Handler{}
assert.Equal(t, "ChartWebhook", handler.Name())
}

76
src/controller/event/metadata/chart.go
View File

@ -1,76 +0,0 @@
package metadata
import (
"time"
event2 "github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/pkg/notifier/event"
)
// ChartMetaData defines meta data of chart event
type ChartMetaData struct {
ProjectName string
ChartName string
Versions []string
OccurAt time.Time
Operator string
}
func (cmd *ChartMetaData) convert(evt *event2.ChartEvent) {
evt.ProjectName = cmd.ProjectName
evt.OccurAt = cmd.OccurAt
evt.Operator = cmd.Operator
evt.ChartName = cmd.ChartName
evt.Versions = cmd.Versions
}
// ChartUploadMetaData defines meta data of chart upload event
type ChartUploadMetaData struct {
ChartMetaData
}
// Resolve chart uploading metadata into common chart event
func (cu *ChartUploadMetaData) Resolve(event *event.Event) error {
data := &event2.ChartEvent{
EventType: event2.TopicUploadChart,
}
cu.convert(data)
event.Topic = event2.TopicUploadChart
event.Data = data
return nil
}
// ChartDownloadMetaData defines meta data of chart download event
type ChartDownloadMetaData struct {
ChartMetaData
}
// Resolve chart download metadata into common chart event
func (cd *ChartDownloadMetaData) Resolve(evt *event.Event) error {
data := &event2.ChartEvent{
EventType: event2.TopicDownloadChart,
}
cd.convert(data)
evt.Topic = event2.TopicDownloadChart
evt.Data = data
return nil
}
// ChartDeleteMetaData defines meta data of chart delete event
type ChartDeleteMetaData struct {
ChartMetaData
}
// Resolve chart delete metadata into common chart event
func (cd *ChartDeleteMetaData) Resolve(evt *event.Event) error {
data := &event2.ChartEvent{
EventType: event2.TopicDeleteChart,
}
cd.convert(data)
evt.Topic = event2.TopicDeleteChart
evt.Data = data
return nil
}

73
src/controller/event/metadata/chart_test.go
View File

@ -1,73 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package metadata
import (
"testing"
"time"
"github.com/stretchr/testify/suite"
event2 "github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/pkg/notifier/event"
)
type chartEventTestSuite struct {
suite.Suite
}
func (r *chartEventTestSuite) TestResolveOfUploadChartEventMetadata() {
e := &event.Event{}
metadata := &ChartUploadMetaData{
ChartMetaData{
ProjectName: "library",
ChartName: "redis-v2.0",
Versions: nil,
OccurAt: time.Time{},
Operator: "admin",
},
}
err := metadata.Resolve(e)
r.Require().Nil(err)
r.Equal(event2.TopicUploadChart, e.Topic)
r.Require().NotNil(e.Data)
data, ok := e.Data.(*event2.ChartEvent)
r.Require().True(ok)
r.Equal("redis-v2.0", data.ChartName)
}
func (r *chartEventTestSuite) TestResolveOfDownloadChartEventMetadata() {
e := &event.Event{}
metadata := &ChartDownloadMetaData{
ChartMetaData{
ProjectName: "library",
ChartName: "redis-v2.0",
Versions: nil,
OccurAt: time.Time{},
Operator: "admin",
},
}
err := metadata.Resolve(e)
r.Require().Nil(err)
r.Equal(event2.TopicDownloadChart, e.Topic)
r.Require().NotNil(e.Data)
data, ok := e.Data.(*event2.ChartEvent)
r.Require().True(ok)
r.Equal("redis-v2.0", data.ChartName)
}
func TestChartEventTestSuite(t *testing.T) {
suite.Run(t, &chartEventTestSuite{})
}

18
src/controller/event/topic.go
View File

@ -44,9 +44,6 @@ const (
// QuotaExceedTopic is topic for quota warning event, the usage reaches the warning bar of limitation, like 85%
TopicQuotaWarning = "QUOTA_WARNING"
TopicQuotaExceed = "QUOTA_EXCEED"
TopicUploadChart = "UPLOAD_CHART"
TopicDownloadChart = "DOWNLOAD_CHART"
TopicDeleteChart = "DELETE_CHART"
TopicReplication = "REPLICATION"
TopicArtifactLabeled = "ARTIFACT_LABELED"
TopicTagRetention = "TAG_RETENTION"
@ -302,21 +299,6 @@ func (s *ScanImageEvent) String() string {
s.Artifact, s.Operator, s.OccurAt.Format("2006-01-02 15:04:05"))
}
// ChartEvent is chart related event data to publish
type ChartEvent struct {
EventType string
ProjectName string
ChartName string
Versions []string
OccurAt time.Time
Operator string
}
func (c *ChartEvent) String() string {
return fmt.Sprintf("ProjectName-%s ChartName-%s Versions-%s Operator-%s OccurAt-%s",
c.ProjectName, c.ChartName, c.Versions, c.Operator, c.OccurAt.Format("2006-01-02 15:04:05"))
}
// QuotaEvent is project quota related event data to publish
type QuotaEvent struct {
EventType string

15
src/controller/health/checker.go
View File

@ -146,18 +146,6 @@ func registryCtlHealthChecker() health.Checker {
return PeriodicHealthChecker(checker, period)
}
func chartmuseumHealthChecker() health.Checker {
url, err := config.GetChartMuseumEndpoint()
if err != nil {
log.Errorf("failed to get the URL of chartmuseum: %v", err)
}
url = url + "/health"
timeout := 60 * time.Second
period := 10 * time.Second
checker := HTTPStatusCodeHealthChecker(http.MethodGet, url, nil, timeout, http.StatusOK)
return PeriodicHealthChecker(checker, period)
}
func notaryHealthChecker() health.Checker {
url := config.InternalNotaryEndpoint() + "/_notary_server/health"
timeout := 60 * time.Second
@ -203,9 +191,6 @@ func RegisterHealthCheckers() {
registry["registryctl"] = registryCtlHealthChecker()
registry["database"] = databaseHealthChecker()
registry["redis"] = redisHealthChecker()
if config.WithChartMuseum() {
registry["chartmuseum"] = chartmuseumHealthChecker()
}
if config.WithNotary() {
registry["notary"] = notaryHealthChecker()
}

11
src/controller/replication/flow/copy_test.go
View File

@ -42,16 +42,6 @@ func (c *copyFlowTestSuite) TestRun() {
},
}, nil)
adp.On("FetchArtifacts", mock.Anything).Return([]*model.Resource{
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: "library/hello-world",
},
Vtags: []string{"latest"},
},
Override: false,
},
{
Type: model.ResourceTypeArtifact,
Metadata: &model.ResourceMetadata{
@ -61,7 +51,6 @@ func (c *copyFlowTestSuite) TestRun() {
Vtags: []string{"latest"},
},
Override: false,
Skip: true,
},
}, nil)
adp.On("PrepareForPush", mock.Anything).Return(nil)

55
src/controller/replication/flow/stage.go
View File

@ -56,57 +56,16 @@ func initialize(policy *repctlmodel.Policy) (adp.Adapter, adp.Adapter, error) {
// fetch resources from the source registry
func fetchResources(adapter adp.Adapter, policy *repctlmodel.Policy) ([]*model.Resource, error) {
var resTypes []string
for _, filter := range policy.Filters {
if filter.Type == model.FilterTypeResource {
resTypes = append(resTypes, filter.Value.(string))
}
}
if len(resTypes) == 0 {
info, err := adapter.Info()
if err != nil {
return nil, fmt.Errorf("failed to get the adapter info: %v", err)
}
resTypes = append(resTypes, info.SupportedResourceTypes...)
}
fetchArtifact := false
fetchChart := false
for _, resType := range resTypes {
if resType == model.ResourceTypeChart {
fetchChart = true
continue
}
fetchArtifact = true
}
var resources []*model.Resource
// artifacts
if fetchArtifact {
reg, ok := adapter.(adp.ArtifactRegistry)
if !ok {
return nil, fmt.Errorf("the adapter doesn't implement the ArtifactRegistry interface")
}
res, err := reg.FetchArtifacts(policy.Filters)
if err != nil {
return nil, fmt.Errorf("failed to fetch artifacts: %v", err)
}
resources = append(resources, res...)
log.Debug("fetch artifacts completed")
reg, ok := adapter.(adp.ArtifactRegistry)
if !ok {
return nil, fmt.Errorf("the adapter doesn't implement the ArtifactRegistry interface")
}
// charts
if fetchChart {
reg, ok := adapter.(adp.ChartRegistry)
if !ok {
return nil, fmt.Errorf("the adapter doesn't implement the ChartRegistry interface")
}
res, err := reg.FetchCharts(policy.Filters)
if err != nil {
return nil, fmt.Errorf("failed to fetch charts: %v", err)
}
resources = append(resources, res...)
log.Debug("fetch charts completed")
res, err := reg.FetchArtifacts(policy.Filters)
if err != nil {
return nil, fmt.Errorf("failed to fetch artifacts: %v", err)
}
resources = append(resources, res...)
log.Debug("fetch resources from the source registry completed")
return resources, nil

11
src/controller/replication/flow/stage_test.go
View File

@ -53,11 +53,6 @@ func (s *stageTestSuite) TestInitialize() {
func (s *stageTestSuite) TestFetchResources() {
adapter := &mockAdapter{}
adapter.On("Info").Return(&model.RegistryInfo{
SupportedResourceTypes: []string{
model.ResourceTypeArtifact,
},
}, nil)
adapter.On("FetchArtifacts", mock.Anything).Return([]*model.Resource{
{},
{},
@ -72,7 +67,7 @@ func (s *stageTestSuite) TestFetchResources() {
func (s *stageTestSuite) TestAssembleSourceResources() {
resources := []*model.Resource{
{
Type: model.ResourceTypeChart,
Type: model.ResourceTypeArtifact,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: "library/hello-world",
@ -95,7 +90,7 @@ func (s *stageTestSuite) TestAssembleSourceResources() {
func (s *stageTestSuite) TestAssembleDestinationResources() {
resources := []*model.Resource{
{
Type: model.ResourceTypeChart,
Type: model.ResourceTypeArtifact,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: "library/hello-world",
@ -114,7 +109,7 @@ func (s *stageTestSuite) TestAssembleDestinationResources() {
res, err := assembleDestinationResources(resources, policy, "")
s.Require().Nil(err)
s.Len(res, 1)
s.Equal(model.ResourceTypeChart, res[0].Type)
s.Equal(model.ResourceTypeArtifact, res[0].Type)
s.Equal("test/hello-world", res[0].Metadata.Repository.Name)
s.Equal(1, len(res[0].Metadata.Vtags))
s.Equal("latest", res[0].Metadata.Vtags[0])

199
src/controller/replication/transfer/chart/transfer.go
View File

@ -1,199 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package chart
import (
"errors"
trans "github.com/goharbor/harbor/src/controller/replication/transfer"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/reg/adapter"
"github.com/goharbor/harbor/src/pkg/reg/model"
)
func init() {
if err := trans.RegisterFactory(model.ResourceTypeChart, factory); err != nil {
log.Errorf("failed to register transfer factory: %v", err)
}
}
func factory(logger trans.Logger, stopFunc trans.StopFunc) (trans.Transfer, error) {
return &transfer{
logger: logger,
isStopped: stopFunc,
}, nil
}
type chart struct {
name string
version string
contentURL string
}
type transfer struct {
logger trans.Logger
isStopped trans.StopFunc
src adapter.ChartRegistry
dst adapter.ChartRegistry
}
func (t *transfer) Transfer(src *model.Resource, dst *model.Resource, opts *trans.Options) error {
// initialize
if err := t.initialize(src, dst); err != nil {
return err
}
// delete the chart on destination registry
if dst.Deleted {
return t.delete(&chart{
name: dst.Metadata.Repository.Name,
version: dst.Metadata.Artifacts[0].Tags[0],
})
}
var contentURL string
if len(src.ExtendedInfo) > 0 && src.ExtendedInfo["contentURL"] != nil {
contentURL = src.ExtendedInfo["contentURL"].(string)
}
srcChart := &chart{
name: src.Metadata.Repository.Name,
version: src.Metadata.Artifacts[0].Tags[0],
contentURL: contentURL,
}
dstChart := &chart{
name: dst.Metadata.Repository.Name,
version: dst.Metadata.Artifacts[0].Tags[0],
}
// copy the chart from source registry to the destination
return t.copy(srcChart, dstChart, dst.Override, opts)
}
func (t *transfer) initialize(src, dst *model.Resource) error {
// create client for source registry
srcReg, err := createRegistry(src.Registry)
if err != nil {
t.logger.Errorf("failed to create client for source registry: %v", err)
return err
}
t.src = srcReg
t.logger.Infof("client for source registry [type: %s, URL: %s, insecure: %v] created",
src.Registry.Type, src.Registry.URL, src.Registry.Insecure)
// create client for destination registry
dstReg, err := createRegistry(dst.Registry)
if err != nil {
t.logger.Errorf("failed to create client for destination registry: %v", err)
return err
}
t.dst = dstReg
t.logger.Infof("client for destination registry [type: %s, URL: %s, insecure: %v] created",
dst.Registry.Type, dst.Registry.URL, dst.Registry.Insecure)
return nil
}
func createRegistry(reg *model.Registry) (adapter.ChartRegistry, error) {
factory, err := adapter.GetFactory(reg.Type)
if err != nil {
return nil, err
}
ad, err := factory.Create(reg)
if err != nil {
return nil, err
}
registry, ok := ad.(adapter.ChartRegistry)
if !ok {
return nil, errors.New("the adapter doesn't implement the \"ChartRegistry\" interface")
}
return registry, nil
}
func (t *transfer) shouldStop() bool {
isStopped := t.isStopped()
if isStopped {
t.logger.Info("the job is stopped")
}
return isStopped
}
func (t *transfer) copy(src, dst *chart, override bool, opts *trans.Options) error {
if t.shouldStop() {
return nil
}
t.logger.Infof("copying %s:%s(source registry) to %s:%s(destination registry)...",
src.name, src.version, dst.name, dst.version)
// check the existence of the chart on the destination registry
exist, err := t.dst.ChartExist(dst.name, dst.version)
if err != nil {
t.logger.Errorf("failed to check the existence of chart %s:%s on the destination registry: %v", dst.name, dst.version, err)
return err
}
if exist {
// the same name chart exists, but not allowed to override
if !override {
t.logger.Warningf("the same name chart %s:%s exists on the destination registry, but the \"override\" is set to false, skip",
dst.name, dst.version)
return nil
}
// the same name chart exists, but allowed to override
t.logger.Warningf("the same name chart %s:%s exists on the destination registry and the \"override\" is set to true, continue...",
dst.name, dst.version)
}
// copy the chart between the source and destination registries
chart, err := t.src.DownloadChart(src.name, src.version, src.contentURL)
if err != nil {
t.logger.Errorf("failed to download the chart %s:%s: %v", src.name, src.version, err)
return err
}
if opts.Speed > 0 {
t.logger.Infof("limit network speed at %d kb/s", opts.Speed)
chart = trans.NewReader(chart, opts.Speed)
}
defer chart.Close()
if err = t.dst.UploadChart(dst.name, dst.version, chart); err != nil {
t.logger.Errorf("failed to upload the chart %s:%s: %v", dst.name, dst.version, err)
return err
}
t.logger.Infof("copy %s:%s(source registry) to %s:%s(destination registry) completed",
src.name, src.version, dst.name, dst.version)
return nil
}
func (t *transfer) delete(chart *chart) error {
exist, err := t.dst.ChartExist(chart.name, chart.version)
if err != nil {
t.logger.Errorf("failed to check the existence of chart %s:%s on the destination registry: %v", chart.name, chart.version, err)
return err
}
if !exist {
t.logger.Infof("the chart %s:%s doesn't exist on the destination registry, skip",
chart.name, chart.version)
return nil
}
t.logger.Infof("deleting the chart %s:%s on the destination registry...", chart.name, chart.version)
if err := t.dst.DeleteChart(chart.name, chart.version); err != nil {
t.logger.Errorf("failed to delete the chart %s:%s on the destination registry: %v", chart.name, chart.version, err)
return err
}
t.logger.Infof("delete the chart %s:%s on the destination registry completed", chart.name, chart.version)
return nil
}

117
src/controller/replication/transfer/chart/transfer_test.go
View File

@ -1,117 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package chart
import (
"bytes"
"io"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
trans "github.com/goharbor/harbor/src/controller/replication/transfer"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/reg/model"
)
type fakeRegistry struct{}
func (f *fakeRegistry) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) {
return []*model.Resource{
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: "library/harbor",
},
Vtags: []string{"0.2.0"},
},
},
}, nil
}
func (f *fakeRegistry) ChartExist(name, version string) (bool, error) {
return true, nil
}
func (f *fakeRegistry) DownloadChart(name, version, contentURL string) (io.ReadCloser, error) {
r := io.NopCloser(bytes.NewReader([]byte{'a'}))
return r, nil
}
func (f *fakeRegistry) UploadChart(name, version string, chart io.Reader) error {
return nil
}
func (f *fakeRegistry) DeleteChart(name, version string) error {
return nil
}
func TestFactory(t *testing.T) {
tr, err := factory(nil, nil)
require.Nil(t, err)
_, ok := tr.(trans.Transfer)
assert.True(t, ok)
}
func TestShouldStop(t *testing.T) {
// should stop
stopFunc := func() bool { return true }
tr := &transfer{
logger: log.DefaultLogger(),
isStopped: stopFunc,
}
assert.True(t, tr.shouldStop())
// should not stop
stopFunc = func() bool { return false }
tr = &transfer{
isStopped: stopFunc,
}
assert.False(t, tr.shouldStop())
}
func TestCopy(t *testing.T) {
stopFunc := func() bool { return false }
transfer := &transfer{
logger: log.DefaultLogger(),
isStopped: stopFunc,
src: &fakeRegistry{},
dst: &fakeRegistry{},
}
src := &chart{
name: "library/harbor",
version: "0.2.0",
}
dst := &chart{
name: "dest/harbor",
version: "0.2.0",
}
err := transfer.copy(src, dst, true, trans.NewOptions())
assert.Nil(t, err)
}
func TestDelete(t *testing.T) {
stopFunc := func() bool { return false }
transfer := &transfer{
logger: log.DefaultLogger(),
isStopped: stopFunc,
src: &fakeRegistry{},
dst: &fakeRegistry{},
}
chart := &chart{
name: "dest/harbor",
version: "0.2.0",
}
err := transfer.delete(chart)
assert.Nil(t, err)
}

2
src/controller/systeminfo/controller.go
View File

@ -60,7 +60,6 @@ type protectedData struct {
HasCARoot bool
RegistryStorageProviderName string
ReadOnly bool
WithChartMuseum bool
NotificationEnable bool
}
@ -121,7 +120,6 @@ func (c *controller) GetInfo(ctx context.Context, opt Options) (*Data, error) {
res.Protected = &protectedData{
CurrentTime: time.Now(),
WithNotary: config.WithNotary(),
WithChartMuseum: config.WithChartMuseum(),
ReadOnly: config.ReadOnly(ctx),
ExtURL: extURL,
RegistryURL: registryURL,

2
src/controller/systeminfo/controller_test.go
View File

@ -31,7 +31,6 @@ func (s *sysInfoCtlTestSuite) SetupTest() {
common.RegistryStorageProviderName: "filesystem",
common.ReadOnly: false,
common.NotificationEnable: false,
common.WithChartMuseum: false,
common.WithNotary: true,
}
@ -77,7 +76,6 @@ func (s *sysInfoCtlTestSuite) TestGetInfo() {
HasCARoot: true,
RegistryStorageProviderName: "filesystem",
ReadOnly: false,
WithChartMuseum: false,
NotificationEnable: false,
},
},

25
src/core/api/api_test.go
View File

@ -19,7 +19,6 @@ import (
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strconv"
"strings"
@ -29,7 +28,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/goharbor/harbor/src/chartserver"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
common_http "github.com/goharbor/harbor/src/common/http"
@ -38,7 +36,6 @@ import (
"github.com/goharbor/harbor/src/pkg/member"
memberModels "github.com/goharbor/harbor/src/pkg/member/models"
"github.com/goharbor/harbor/src/pkg/user"
htesting "github.com/goharbor/harbor/src/testing"
)
var (
@ -315,25 +312,3 @@ func clean() {
}
}
}
// Provides a mock chart controller for deletable test cases
func mockChartController() (*httptest.Server, *chartserver.Controller, error) {
mockServer := httptest.NewServer(htesting.MockChartRepoHandler)
var oldController, newController *chartserver.Controller
url, err := url.Parse(mockServer.URL)
if err == nil {
newController, err = chartserver.NewController(url)
}
if err != nil {
mockServer.Close()
return nil, nil, err
}
// Override current controller and keep the old one for restoring
oldController = chartController
chartController = newController
return mockServer, oldController, nil
}

21
src/core/api/base.go
View File

@ -30,7 +30,6 @@ import (
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/controller/p2p/preheat"
projectcontroller "github.com/goharbor/harbor/src/controller/project"
"github.com/goharbor/harbor/src/lib/config"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/scheduler"
@ -175,11 +174,6 @@ func (b *BaseController) PopulateUserSession(u models.User) {
// Init related objects/configurations for the API controllers
func Init() error {
// init chart controller
if err := initChartController(); err != nil {
return err
}
p2pPreheatCallbackFun := func(ctx context.Context, p string) error {
param := &preheat.TriggerParam{}
if err := json.Unmarshal([]byte(p), param); err != nil {
@ -192,18 +186,3 @@ func Init() error {
return err
}
func initChartController() error {
// If chart repository is not enabled then directly return
if !config.WithChartMuseum() {
return nil
}
chartCtl, err := initializeChartController()
if err != nil {
return err
}
chartController = chartCtl
return nil
}

114
src/core/api/chart_label.go
View File

@ -1,114 +0,0 @@
package api
import (
"errors"
"fmt"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/pkg/label/model"
proModels "github.com/goharbor/harbor/src/pkg/project/models"
)
const (
versionParam = ":version"
idParam = ":id"
)
// ChartLabelAPI handles the requests of marking/removing labels to/from charts.
type ChartLabelAPI struct {
LabelResourceAPI
project *proModels.Project
chartFullName string
}
// Prepare required material for follow-up actions.
func (cla *ChartLabelAPI) Prepare() {
// Super
cla.LabelResourceAPI.Prepare()
// Check authorization
if !cla.SecurityCtx.IsAuthenticated() {
cla.SendUnAuthorizedError(errors.New("UnAuthorized"))
return
}
project := cla.GetStringFromPath(namespaceParam)
// Project should be a valid existing one
existingProject, err := cla.ProjectCtl.Get(cla.Context(), project)
if err != nil {
cla.SendError(err)
return
}
cla.project = existingProject
// Check the existence of target chart
chartName := cla.GetStringFromPath(nameParam)
version := cla.GetStringFromPath(versionParam)
if _, err = chartController.GetChartVersion(project, chartName, version); err != nil {
cla.SendNotFoundError(err)
return
}
cla.chartFullName = fmt.Sprintf("%s/%s:%s", project, chartName, version)
}
func (cla *ChartLabelAPI) requireAccess(action rbac.Action) bool {
return cla.RequireProjectAccess(cla.project.ProjectID, action, rbac.ResourceHelmChartVersionLabel)
}
// MarkLabel handles the request of marking label to chart.
func (cla *ChartLabelAPI) MarkLabel() {
if !cla.requireAccess(rbac.ActionCreate) {
return
}
l := &model.Label{}
if err := cla.DecodeJSONReq(l); err != nil {
cla.SendBadRequestError(err)
return
}
label, ok := cla.validate(l.ID, cla.project.ProjectID)
if !ok {
return
}
label2Res := &models.ResourceLabel{
LabelID: label.ID,
ResourceType: common.ResourceTypeChart,
ResourceName: cla.chartFullName,
}
cla.markLabelToResource(label2Res)
}
// RemoveLabel handles the request of removing label from chart.
func (cla *ChartLabelAPI) RemoveLabel() {
if !cla.requireAccess(rbac.ActionDelete) {
return
}
lID, err := cla.GetInt64FromPath(idParam)
if err != nil {
cla.SendInternalServerError(err)
return
}
label, ok := cla.exists(lID)
if !ok {
return
}
cla.removeLabelFromResource(common.ResourceTypeChart, cla.chartFullName, label.ID)
}
// GetLabels gets labels for the specified chart version.
func (cla *ChartLabelAPI) GetLabels() {
if !cla.requireAccess(rbac.ActionList) {
return
}
cla.getLabelsOfResource(common.ResourceTypeChart, cla.chartFullName)
}

226
src/core/api/chart_label_test.go
View File

@ -1,226 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/goharbor/harbor/src/chartserver"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/api"
"github.com/goharbor/harbor/src/lib/orm"
pkg_dao "github.com/goharbor/harbor/src/pkg/label/dao"
"github.com/goharbor/harbor/src/pkg/label/model"
)
var (
resourceLabelAPIPath = fmt.Sprintf("/api/%s/chartrepo/library/charts/harbor/0.2.0/labels", api.APIVersion)
resourceLabelAPIPathWithFakeProject = fmt.Sprintf("/api/%s/chartrepo/not-exist/charts/harbor/0.2.0/labels", api.APIVersion)
resourceLabelAPIPathWithFakeChart = fmt.Sprintf("/api/%s/chartrepo/library/charts/not-exist/0.2.0/labels", api.APIVersion)
cProLibraryLabelID int64
mockChartServer *httptest.Server
oldChartController *chartserver.Controller
labelDao pkg_dao.DAO
)
func TestToStartMockChartService(t *testing.T) {
var err error
mockChartServer, oldChartController, err = mockChartController()
if err != nil {
t.Fatalf("failed to start the mock chart service: %v", err)
}
}
func TestAddToChart(t *testing.T) {
labelDao = pkg_dao.New()
cSysLevelLabelID, err := labelDao.Create(orm.Context(), &model.Label{
Name: "c_sys_level_label",
Level: common.LabelLevelSystem,
})
require.Nil(t, err)
defer labelDao.Delete(orm.Context(), cSysLevelLabelID)
cProTestLabelID, err := labelDao.Create(orm.Context(), &model.Label{
Name: "c_pro_test_label",
Level: common.LabelLevelUser,
Scope: common.LabelScopeProject,
ProjectID: 100,
})
require.Nil(t, err)
defer labelDao.Delete(orm.Context(), cProTestLabelID)
cProLibraryLabelID, err = labelDao.Create(orm.Context(), &model.Label{
Name: "c_pro_library_label",
Level: common.LabelLevelUser,
Scope: common.LabelScopeProject,
ProjectID: 1,
})
require.Nil(t, err)
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
url: resourceLabelAPIPath,
method: http.MethodPost,
},
code: http.StatusUnauthorized,
},
// 403
{
request: &testingRequest{
url: resourceLabelAPIPath,
method: http.MethodPost,
credential: projGuest,
},
code: http.StatusForbidden,
},
// 500 project doesn't exist
{
request: &testingRequest{
url: resourceLabelAPIPathWithFakeProject,
method: http.MethodPost,
credential: projDeveloper,
},
code: http.StatusNotFound,
},
// 404 chart doesn't exist
{
request: &testingRequest{
url: resourceLabelAPIPathWithFakeChart,
method: http.MethodPost,
credential: projDeveloper,
},
code: http.StatusNotFound,
},
// 400
{
request: &testingRequest{
url: resourceLabelAPIPath,
method: http.MethodPost,
credential: projDeveloper,
},
code: http.StatusBadRequest,
},
// 404 label doesn't exist
{
request: &testingRequest{
url: resourceLabelAPIPath,
method: http.MethodPost,
credential: projDeveloper,
bodyJSON: struct {
ID int64
}{
ID: 1000,
},
},
code: http.StatusNotFound,
},
// 400 system level label
{
request: &testingRequest{
url: resourceLabelAPIPath,
method: http.MethodPost,
credential: projDeveloper,
bodyJSON: struct {
ID int64
}{
ID: cSysLevelLabelID,
},
},
code: http.StatusBadRequest,
},
// 400 try to add the label of project1 to the image under project2
{
request: &testingRequest{
url: resourceLabelAPIPath,
method: http.MethodPost,
credential: projDeveloper,
bodyJSON: struct {
ID int64
}{
ID: cProTestLabelID,
},
},
code: http.StatusBadRequest,
},
// 200
{
request: &testingRequest{
url: resourceLabelAPIPath,
method: http.MethodPost,
credential: projDeveloper,
bodyJSON: struct {
ID int64
}{
ID: cProLibraryLabelID,
},
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}
func TestGetOfChart(t *testing.T) {
labels := []*model.Label{}
err := handleAndParse(&testingRequest{
url: resourceLabelAPIPath,
method: http.MethodGet,
credential: projDeveloper,
}, &labels)
require.Nil(t, err)
require.Equal(t, 1, len(labels))
assert.Equal(t, cProLibraryLabelID, labels[0].ID)
}
func TestRemoveFromChart(t *testing.T) {
runCodeCheckingCases(t, &codeCheckingCase{
request: &testingRequest{
url: fmt.Sprintf("%s/%d", resourceLabelAPIPath, cProLibraryLabelID),
method: http.MethodDelete,
credential: projDeveloper,
},
code: http.StatusOK,
})
labels := []*model.Label{}
err := handleAndParse(&testingRequest{
url: resourceLabelAPIPath,
method: http.MethodGet,
credential: projDeveloper,
}, &labels)
require.Nil(t, err)
require.Equal(t, 0, len(labels))
}
func TestToStopMockChartService(t *testing.T) {
if mockChartServer != nil {
mockChartServer.Close()
}
if oldChartController != nil {
chartController = oldChartController
}
labelDao = pkg_dao.New()
labelDao.Delete(orm.Context(), cProLibraryLabelID)
}

651
src/core/api/chart_repository.go
View File

@ -1,651 +0,0 @@
package api
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"time"
"github.com/goharbor/harbor/src/chartserver"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/rbac"
rep_event "github.com/goharbor/harbor/src/controller/event/handler/replication/event"
"github.com/goharbor/harbor/src/controller/event/metadata"
"github.com/goharbor/harbor/src/controller/project"
"github.com/goharbor/harbor/src/core/label"
"github.com/goharbor/harbor/src/lib/config"
hlog "github.com/goharbor/harbor/src/lib/log"
pkg_label "github.com/goharbor/harbor/src/pkg/label"
n_event "github.com/goharbor/harbor/src/pkg/notifier/event"
"github.com/goharbor/harbor/src/pkg/reg/model"
"github.com/goharbor/harbor/src/server/middleware/orm"
)
const (
namespaceParam = ":repo"
nameParam = ":name"
filenameParam = ":filename"
defaultRepo = "library"
rootUploadingEndpoint = "/api/chartrepo/charts"
rootIndexEndpoint = "/chartrepo/index.yaml"
chartRepoHealthEndpoint = "/api/chartrepo/health"
formFieldNameForChart = "chart"
formFiledNameForProv = "prov"
headerContentType = "Content-Type"
contentTypeMultipart = "multipart/form-data"
// chartPackageFileExtension is the file extension used for chart packages
chartPackageFileExtension = "tgz"
)
// chartController is a singleton instance
var chartController *chartserver.Controller
// GetChartController returns the chart controller
func GetChartController() *chartserver.Controller {
return chartController
}
// ChartRepositoryAPI provides related API handlers for the chart repository APIs
type ChartRepositoryAPI struct {
// The base controller to provide common utilities
BaseController
// For label management
labelManager *label.BaseManager
// Keep the namespace if existing
namespace string
}
// Prepare something for the following actions
func (cra *ChartRepositoryAPI) Prepare() {
// Call super prepare method
cra.BaseController.Prepare()
// Try to extract namespace for parameter of path
// It may not exist
cra.namespace = strings.TrimSpace(cra.GetStringFromPath(namespaceParam))
// Check the existence of namespace
// Exclude the following URI
// -/index.yaml
// -/api/chartserver/health
incomingURI := cra.Ctx.Request.URL.Path
if incomingURI == rootUploadingEndpoint {
// Forward to the default repository
cra.namespace = defaultRepo
}
if incomingURI != rootIndexEndpoint &&
incomingURI != chartRepoHealthEndpoint {
if !cra.requireNamespace(cra.namespace) {
return
}
}
// Init label manager
cra.labelManager = &label.BaseManager{
LabelMgr: pkg_label.Mgr,
}
}
func (cra *ChartRepositoryAPI) requireAccess(action rbac.Action, subresource ...rbac.Resource) bool {
if len(subresource) == 0 {
subresource = append(subresource, rbac.ResourceHelmChart)
}
return cra.RequireProjectAccess(cra.namespace, action, subresource...)
}
// GetHealthStatus handles GET /api/chartrepo/health
func (cra *ChartRepositoryAPI) GetHealthStatus() {
// Check access
if !cra.SecurityCtx.IsAuthenticated() {
cra.SendUnAuthorizedError(errors.New("unauthorized"))
return
}
if !cra.SecurityCtx.IsSysAdmin() {
cra.SendForbiddenError(errors.New(cra.SecurityCtx.GetUsername()))
return
}
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// GetIndexByRepo handles GET /:repo/index.yaml
func (cra *ChartRepositoryAPI) GetIndexByRepo() {
// Check access
if !cra.requireAccess(rbac.ActionRead) {
return
}
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// GetIndex handles GET /index.yaml
func (cra *ChartRepositoryAPI) GetIndex() {
// Check access
if !cra.SecurityCtx.IsAuthenticated() {
cra.SendUnAuthorizedError(errors.New("unauthorized"))
return
}
if !cra.SecurityCtx.IsSysAdmin() {
cra.SendForbiddenError(errors.New(cra.SecurityCtx.GetUsername()))
return
}
projects, err := cra.ProjectCtl.List(cra.Context(), nil, project.Metadata(false))
if err != nil {
cra.SendInternalServerError(err)
return
}
namespaces := []string{}
for _, r := range projects {
namespaces = append(namespaces, r.Name)
}
indexFile, err := chartController.GetIndexFile(namespaces)
if err != nil {
cra.SendInternalServerError(err)
return
}
cra.WriteYamlData(indexFile)
}
// DownloadChart handles GET /:repo/charts/:filename
func (cra *ChartRepositoryAPI) DownloadChart() {
// Check access
if !cra.requireAccess(rbac.ActionRead) {
return
}
namespace := cra.GetStringFromPath(namespaceParam)
fileName := cra.GetStringFromPath(filenameParam)
// Add hook event to request context
cra.addDownloadChartEventContext(fileName, namespace, cra.Ctx.Request)
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// ListCharts handles GET /api/:repo/charts
func (cra *ChartRepositoryAPI) ListCharts() {
// Check access
if !cra.requireAccess(rbac.ActionList) {
return
}
charts, err := chartController.ListCharts(cra.namespace)
if err != nil {
cra.ParseAndHandleError("fail to list charts", err)
return
}
cra.WriteJSONData(charts)
}
// ListChartVersions GET /api/:repo/charts/:name
func (cra *ChartRepositoryAPI) ListChartVersions() {
// Check access
if !cra.requireAccess(rbac.ActionList, rbac.ResourceHelmChartVersion) {
return
}
chartName := cra.GetStringFromPath(nameParam)
versions, err := chartController.GetChart(cra.namespace, chartName)
if err != nil {
cra.ParseAndHandleError("fail to get chart", err)
return
}
// Append labels
for _, chartVersion := range versions {
labels, err := cra.labelManager.GetLabelsOfResource(common.ResourceTypeChart, chartFullName(cra.namespace, chartVersion.Name, chartVersion.Version))
if err != nil {
cra.SendInternalServerError(err)
return
}
chartVersion.Labels = labels
}
cra.WriteJSONData(versions)
}
// GetChartVersion handles GET /api/:repo/charts/:name/:version
func (cra *ChartRepositoryAPI) GetChartVersion() {
// Check access
if !cra.requireAccess(rbac.ActionRead, rbac.ResourceHelmChartVersion) {
return
}
// Get other parameters
chartName := cra.GetStringFromPath(nameParam)
version := cra.GetStringFromPath(versionParam)
chartVersion, err := chartController.GetChartVersionDetails(cra.namespace, chartName, version)
if err != nil {
cra.ParseAndHandleError("fail to get chart version", err)
return
}
// Append labels
labels, err := cra.labelManager.GetLabelsOfResource(common.ResourceTypeChart, chartFullName(cra.namespace, chartName, version))
if err != nil {
cra.SendInternalServerError(err)
return
}
chartVersion.Labels = labels
cra.WriteJSONData(chartVersion)
}
// DeleteChartVersion handles DELETE /api/:repo/charts/:name/:version
func (cra *ChartRepositoryAPI) DeleteChartVersion() {
// Check access
if !cra.requireAccess(rbac.ActionDelete, rbac.ResourceHelmChartVersion) {
return
}
// Get other parameters
chartName := cra.GetStringFromPath(nameParam)
version := cra.GetStringFromPath(versionParam)
// Try to remove labels from deleting chart if existing
if err := cra.removeLabelsFromChart(chartName, version); err != nil {
cra.SendInternalServerError(err)
return
}
if err := chartController.DeleteChartVersion(cra.namespace, chartName, version); err != nil {
cra.ParseAndHandleError("fail to delete chart version", err)
return
}
event := &n_event.Event{}
metaData := &metadata.ChartDeleteMetaData{
ChartMetaData: metadata.ChartMetaData{
ProjectName: cra.namespace,
ChartName: chartName,
Versions: []string{version},
OccurAt: time.Now(),
Operator: cra.SecurityCtx.GetUsername(),
},
}
if err := event.Build(metaData); err == nil {
if err := event.Publish(); err != nil {
hlog.Errorf("failed to publish chart delete event: %v", err)
}
} else {
hlog.Errorf("failed to build chart delete event metadata: %v", err)
}
}
// UploadChartVersion handles POST /api/:repo/charts
func (cra *ChartRepositoryAPI) UploadChartVersion() {
hlog.Debugf("Header of request of uploading chart: %#v, content-len=%d", cra.Ctx.Request.Header, cra.Ctx.Request.ContentLength)
// Check access
if !cra.requireAccess(rbac.ActionCreate, rbac.ResourceHelmChartVersion) {
return
}
// Rewrite file content if the content type is "multipart/form-data"
if isMultipartFormData(cra.Ctx.Request) {
formFiles := make([]formFile, 0)
formFiles = append(formFiles,
formFile{
formField: formFieldNameForChart,
mustHave: true,
},
formFile{
formField: formFiledNameForProv,
})
if err := cra.rewriteFileContent(formFiles, cra.Ctx.Request); err != nil {
cra.SendInternalServerError(err)
return
}
if err := cra.addEventContext(formFiles, cra.Ctx.Request); err != nil {
hlog.Errorf("Failed to add chart upload context, %v", err)
}
}
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// UploadChartProvFile handles POST /api/:repo/prov
func (cra *ChartRepositoryAPI) UploadChartProvFile() {
// Check access
if !cra.requireAccess(rbac.ActionCreate) {
return
}
// Rewrite file content if the content type is "multipart/form-data"
if isMultipartFormData(cra.Ctx.Request) {
formFiles := make([]formFile, 0)
formFiles = append(formFiles,
formFile{
formField: formFiledNameForProv,
mustHave: true,
})
if err := cra.rewriteFileContent(formFiles, cra.Ctx.Request); err != nil {
cra.SendInternalServerError(err)
return
}
}
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}
// DeleteChart deletes all the chart versions of the specified chart.
func (cra *ChartRepositoryAPI) DeleteChart() {
// Check access
if !cra.requireAccess(rbac.ActionDelete) {
return
}
// Get other parameters from the request
chartName := cra.GetStringFromPath(nameParam)
// Remove labels from all the deleting chart versions under the chart
chartVersions, err := chartController.GetChart(cra.namespace, chartName)
if err != nil {
cra.ParseAndHandleError("fail to get chart", err)
return
}
versions := []string{}
for _, chartVersion := range chartVersions {
versions = append(versions, chartVersion.Version)
if err := cra.removeLabelsFromChart(chartName, chartVersion.Version); err != nil {
cra.SendInternalServerError(err)
return
}
}
if err := chartController.DeleteChart(cra.namespace, chartName); err != nil {
cra.SendInternalServerError(err)
return
}
event := &n_event.Event{}
metaData := &metadata.ChartDeleteMetaData{
ChartMetaData: metadata.ChartMetaData{
ProjectName: cra.namespace,
ChartName: chartName,
Versions: versions,
OccurAt: time.Now(),
Operator: cra.SecurityCtx.GetUsername(),
},
}
if err := event.Build(metaData); err == nil {
if err := event.Publish(); err != nil {
hlog.Errorf("failed to publish chart delete event: %v", err)
}
} else {
hlog.Errorf("failed to build chart delete event metadata: %v", err)
}
}
func (cra *ChartRepositoryAPI) removeLabelsFromChart(chartName, version string) error {
// Try to remove labels from deleting chart if existing
resourceID := chartFullName(cra.namespace, chartName, version)
labels, err := cra.labelManager.GetLabelsOfResource(common.ResourceTypeChart, resourceID)
if err == nil && len(labels) > 0 {
for _, l := range labels {
if err := cra.labelManager.RemoveLabelFromResource(common.ResourceTypeChart, resourceID, l.ID); err != nil {
return err
}
}
}
return nil
}
// Check if there exists a valid namespace
// Return true if it does
// Return false if it does not
func (cra *ChartRepositoryAPI) requireNamespace(namespace string) bool {
// Actually, never should be like this
if len(namespace) == 0 {
cra.SendBadRequestError(errors.New(":repo should be in the request URL"))
return false
}
existing, err := cra.ProjectCtl.Exists(cra.Context(), namespace)
if err != nil {
// Check failed with error
cra.SendInternalServerError(fmt.Errorf("failed to check existence of namespace %s with error: %s", namespace, err.Error()))
return false
}
// Not existing
if !existing {
cra.handleProjectNotFound(namespace)
return false
}
return true
}
// formFile is used to represent the uploaded files in the form
type formFile struct {
// form field key contains the form file
formField string
// flag to indicate if the file identified by the 'formField'
// must exist
mustHave bool
}
// The func is for event based chart replication policy.
// It will add a context for uploading request with key chart_upload, and consumed by upload response.
func (cra *ChartRepositoryAPI) addEventContext(files []formFile, request *http.Request) error {
if len(files) == 0 {
return nil
}
for _, f := range files {
if f.formField == formFieldNameForChart {
mFile, _, err := cra.GetFile(f.formField)
if err != nil {
hlog.Errorf("failed to read file content for upload event, %v", err)
return err
}
var Buf bytes.Buffer
_, err = io.Copy(&Buf, mFile)
if err != nil {
hlog.Errorf("failed to copy file content for upload event, %v", err)
return err
}
chartOpr := chartserver.ChartOperator{}
chartDetails, err := chartOpr.GetChartData(Buf.Bytes())
if err != nil {
hlog.Errorf("failed to get chart content for upload event, %v", err)
return err
}
extInfo := make(map[string]interface{})
extInfo["operator"] = cra.SecurityCtx.GetUsername()
extInfo["projectName"] = cra.namespace
extInfo["chartName"] = chartDetails.Metadata.Name
var public bool
project, err := cra.ProjectCtl.Get(cra.Context(), cra.namespace)
if err != nil {
hlog.Errorf("failed to check the public of project %s: %v", cra.namespace, err)
public = false
} else {
public = project.IsPublic()
}
e := &rep_event.Event{
Type: rep_event.EventTypeChartUpload,
Resource: &model.Resource{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: fmt.Sprintf("%s/%s", cra.namespace, chartDetails.Metadata.Name),
Metadata: map[string]interface{}{
"public": strconv.FormatBool(public),
},
},
Artifacts: []*model.Artifact{
{
Tags: []string{chartDetails.Metadata.Version},
},
},
},
ExtendedInfo: extInfo,
},
}
*request = *(request.WithContext(context.WithValue(request.Context(), common.ChartUploadCtxKey, e)))
break
}
}
return nil
}
func (cra *ChartRepositoryAPI) addDownloadChartEventContext(fileName, namespace string, request *http.Request) {
chartName, version := parseChartVersionFromFilename(fileName)
event := &metadata.ChartDownloadMetaData{
ChartMetaData: metadata.ChartMetaData{
ProjectName: namespace,
ChartName: chartName,
Versions: []string{version},
OccurAt: time.Now(),
Operator: cra.SecurityCtx.GetUsername(),
},
}
*request = *(request.WithContext(context.WithValue(request.Context(), common.ChartDownloadCtxKey, event)))
}
// If the files are uploaded with multipart/form-data mimetype, beego will extract the data
// from the request automatically. Then the request passed to the backend server with proxying
// way will have empty content.
// This method will refill the requests with file content.
func (cra *ChartRepositoryAPI) rewriteFileContent(files []formFile, request *http.Request) error {
if len(files) == 0 {
return nil // no files, early return
}
var body bytes.Buffer
w := multipart.NewWriter(&body)
defer func() {
if err := w.Close(); err != nil {
// Just log it
hlog.Errorf("Failed to defer close multipart writer with error: %s", err.Error())
}
}()
// Process files by key one by one
for _, f := range files {
mFile, mHeader, err := cra.GetFile(f.formField)
// Handle error case by case
if err != nil {
formatedErr := fmt.Errorf("get file content with multipart header from key '%s' failed with error: %s", f.formField, err.Error())
if f.mustHave || err != http.ErrMissingFile {
return formatedErr
}
// Error can be ignored, just log it
hlog.Warning(formatedErr.Error())
continue
}
fw, err := w.CreateFormFile(f.formField, mHeader.Filename)
if err != nil {
return fmt.Errorf("create form file with multipart header failed with error: %s", err.Error())
}
_, err = io.Copy(fw, mFile)
if err != nil {
return fmt.Errorf("copy file stream in multipart form data failed with error: %s", err.Error())
}
}
request.Header.Set(headerContentType, w.FormDataContentType())
request.ContentLength = -1
request.Body = io.NopCloser(&body)
return nil
}
// Initialize the chart service controller
func initializeChartController() (*chartserver.Controller, error) {
addr, err := config.GetChartMuseumEndpoint()
if err != nil {
return nil, fmt.Errorf("failed to get the endpoint URL of chart storage server: %s", err.Error())
}
addr = strings.TrimSuffix(addr, "/")
url, err := url.Parse(addr)
if err != nil {
return nil, errors.New("endpoint URL of chart storage server is malformed")
}
controller, err := chartserver.NewController(url, orm.Middleware())
if err != nil {
return nil, errors.New("failed to initialize chart API controller")
}
hlog.Debugf("Chart storage server is set to %s", url.String())
hlog.Info("API controller for chart repository server is successfully initialized")
return controller, nil
}
// Check if the request content type is "multipart/form-data"
func isMultipartFormData(req *http.Request) bool {
return strings.Contains(req.Header.Get(headerContentType), contentTypeMultipart)
}
// Return the chart full name
func chartFullName(namespace, chartName, version string) string {
if strings.HasPrefix(chartName, "http") {
return fmt.Sprintf("%s:%s", chartName, version)
}
return fmt.Sprintf("%s/%s:%s", namespace, chartName, version)
}
// parseChartVersionFromFilename parse chart and version from file name
func parseChartVersionFromFilename(filename string) (string, string) {
noExt := strings.TrimSuffix(path.Base(filename), fmt.Sprintf(".%s", chartPackageFileExtension))
parts := strings.Split(noExt, "-")
lastIndex := len(parts) - 1
name := parts[0]
version := ""
for idx := lastIndex; idx >= 1; idx-- {
if _, err := strconv.Atoi(string(parts[idx][0])); err == nil { // see if this part looks like a version (starts w int)
version = strings.Join(parts[idx:], "-")
name = strings.Join(parts[:idx], "-")
break
}
}
if version == "" { // no parts looked like a real version, just take everything after last hyphen
name = strings.Join(parts[:lastIndex], "-")
version = parts[lastIndex]
}
return name, version
}

234
src/core/api/chart_repository_test.go
View File

@ -1,234 +0,0 @@
package api
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"testing"
bcontext "github.com/beego/beego/v2/server/web/context"
"github.com/goharbor/harbor/src/chartserver"
proModels "github.com/goharbor/harbor/src/pkg/project/models"
projecttesting "github.com/goharbor/harbor/src/testing/controller/project"
"github.com/goharbor/harbor/src/testing/mock"
)
var (
crOldController *chartserver.Controller
crMockServer *httptest.Server
)
func TestIsMultipartFormData(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/api/chartrepo/charts", nil)
req.Header.Set(headerContentType, "application/json")
if isMultipartFormData(req) {
t.Fatal("expect false result but got true")
}
req.Header.Set(headerContentType, contentTypeMultipart)
if !isMultipartFormData(req) {
t.Fatalf("expect %s result but got %s", contentTypeMultipart, req.Header.Get(headerContentType))
}
}
// Test namespace cheking
func TestRequireNamespace(t *testing.T) {
chartAPI := &ChartRepositoryAPI{}
chartAPI.Ctx = bcontext.NewContext()
chartAPI.Ctx.Request = httptest.NewRequest("GET", "/", nil)
projectCtl := &projecttesting.Controller{}
chartAPI.ProjectCtl = projectCtl
mock.OnAnything(projectCtl, "List").Return([]*proModels.Project{
{ProjectID: 0, Name: "library"},
{ProjectID: 1, Name: "repo2"},
}, nil)
mock.OnAnything(projectCtl, "Exists").Return(
func(ctx context.Context, projectIDOrName interface{}) bool {
if projectIDOrName == nil {
return false
}
if ns, ok := projectIDOrName.(string); ok {
if ns == "library" {
return true
}
return false
}
return false
},
func(ctx context.Context, projectIDOrName interface{}) error {
if projectIDOrName == nil {
return errors.New("nil projectIDOrName")
}
if _, ok := projectIDOrName.(string); ok {
return nil
}
return errors.New("unknown type of projectIDOrName")
},
)
if !chartAPI.requireNamespace("library") {
t.Fatal("expect namespace 'library' existing but got false")
}
}
// Prepare
func TestPrepareEnv(t *testing.T) {
var err error
crMockServer, crOldController, err = mockChartController()
if err != nil {
t.Fatalf("Failed to start mock chart service with error: %s", err)
}
}
// Test get health
func TestGetHealthStatus(t *testing.T) {
status := make(map[string]interface{})
err := handleAndParse(&testingRequest{
url: "/api/chartrepo/health",
method: http.MethodGet,
credential: sysAdmin,
}, &status)
if err != nil {
t.Fatal(err)
}
if _, ok := status["health"]; !ok {
t.Fatal("expect 'health' but got nil")
}
}
// Test get index by repo
func TestGetIndexByRepo(t *testing.T) {
runCodeCheckingCases(t, &codeCheckingCase{
request: &testingRequest{
url: "/chartrepo/library/index.yaml",
method: http.MethodGet,
credential: projDeveloper,
},
code: http.StatusOK,
})
}
// Test get index
func TestGetIndex(t *testing.T) {
runCodeCheckingCases(t, &codeCheckingCase{
request: &testingRequest{
url: "/chartrepo/index.yaml",
method: http.MethodGet,
credential: sysAdmin,
},
code: http.StatusOK,
})
}
// Test download chart
func TestDownloadChart(t *testing.T) {
runCodeCheckingCases(t, &codeCheckingCase{
request: &testingRequest{
url: "/chartrepo/library/charts/harbor-0.2.0.tgz",
method: http.MethodGet,
credential: projDeveloper,
},
code: http.StatusOK,
})
}
// Test get charts
func TesListCharts(t *testing.T) {
charts := make([]*chartserver.ChartInfo, 0)
err := handleAndParse(&testingRequest{
url: "/api/chartrepo/library/charts",
method: http.MethodGet,
credential: projAdmin,
}, &charts)
if err != nil {
t.Fatal(err)
}
if len(charts) != 2 {
t.Fatalf("expect 2 charts but got %d", len(charts))
}
}
// Test get chart versions
func TestListChartVersions(t *testing.T) {
chartVersions := make(chartserver.ChartVersions, 0)
err := handleAndParse(&testingRequest{
url: "/api/chartrepo/library/charts/harbor",
method: http.MethodGet,
credential: projAdmin,
}, &chartVersions)
if err != nil {
t.Fatal(err)
}
if len(chartVersions) != 2 {
t.Fatalf("expect 2 chart versions but got %d", len(chartVersions))
}
}
// Test get chart version details
func TestGetChartVersion(t *testing.T) {
chartV := &chartserver.ChartVersionDetails{}
err := handleAndParse(&testingRequest{
url: "/api/chartrepo/library/charts/harbor/0.2.0",
method: http.MethodGet,
credential: projAdmin,
}, chartV)
if err != nil {
t.Fatal(err)
}
if chartV.Metadata.Name != "harbor" {
t.Fatalf("expect get chart 'harbor' but got %s", chartV.Metadata.Name)
}
if chartV.Metadata.Version != "0.2.0" {
t.Fatalf("expect get chart version '0.2.0' but got %s", chartV.Metadata.Version)
}
}
// Test delete chart version
func TestDeleteChartVersion(t *testing.T) {
runCodeCheckingCases(t, &codeCheckingCase{
request: &testingRequest{
url: "/api/chartrepo/library/charts/harbor/0.2.1",
method: http.MethodDelete,
credential: projAdmin,
},
code: http.StatusOK,
})
}
// Test delete chart
func TestDeleteChart(t *testing.T) {
runCodeCheckingCases(t, &codeCheckingCase{
request: &testingRequest{
url: "/api/chartrepo/library/charts/harbor",
method: http.MethodDelete,
credential: projAdmin,
},
code: http.StatusOK,
})
}
// Clear
func TestClearEnv(t *testing.T) {
crMockServer.Close()
chartController = crOldController
}

22
src/core/api/harborapi_test.go
View File

@ -25,7 +25,6 @@ import (
"github.com/beego/beego/v2/server/web"
"github.com/dghubble/sling"
"github.com/goharbor/harbor/src/common/api"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/job/test"
testutils "github.com/goharbor/harbor/src/common/utils/test"
@ -87,27 +86,6 @@ func init() {
web.BConfig.WebConfig.Session.SessionOn = true
web.TestBeegoInit(apppath)
// Charts are controlled under projects
chartRepositoryAPIType := &ChartRepositoryAPI{}
web.Router("/api/chartrepo/health", chartRepositoryAPIType, "get:GetHealthStatus")
web.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "get:ListCharts")
web.Router("/api/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "get:ListChartVersions")
web.Router("/api/chartrepo/:repo/charts/:name", chartRepositoryAPIType, "delete:DeleteChart")
web.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "get:GetChartVersion")
web.Router("/api/chartrepo/:repo/charts/:name/:version", chartRepositoryAPIType, "delete:DeleteChartVersion")
web.Router("/api/chartrepo/:repo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
web.Router("/api/chartrepo/:repo/prov", chartRepositoryAPIType, "post:UploadChartProvFile")
web.Router("/api/chartrepo/charts", chartRepositoryAPIType, "post:UploadChartVersion")
// Repository services
web.Router("/chartrepo/:repo/index.yaml", chartRepositoryAPIType, "get:GetIndexByRepo")
web.Router("/chartrepo/index.yaml", chartRepositoryAPIType, "get:GetIndex")
web.Router("/chartrepo/:repo/charts/:filename", chartRepositoryAPIType, "get:DownloadChart")
// Labels for chart
chartLabelAPIType := &ChartLabelAPI{}
web.Router("/api/"+api.APIVersion+"/chartrepo/:repo/charts/:name/:version/labels", chartLabelAPIType, "get:GetLabels;post:MarkLabel")
web.Router("/api/"+api.APIVersion+"/chartrepo/:repo/charts/:name/:version/labels/:id([0-9]+)", chartLabelAPIType, "delete:RemoveLabel")
web.Router("/api/internal/syncquota", &InternalAPI{}, "post:SyncQuota")
// Init user Info

31
src/jobservice/job/impl/replication/replication.go
View File

@ -20,10 +20,37 @@ import (
"github.com/goharbor/harbor/src/controller/replication/transfer"
// import chart transfer
_ "github.com/goharbor/harbor/src/controller/replication/transfer/chart"
// import image transfer
_ "github.com/goharbor/harbor/src/controller/replication/transfer/image"
"github.com/goharbor/harbor/src/jobservice/job"
// import aliacr adapter
_ "github.com/goharbor/harbor/src/pkg/reg/adapter/aliacr"
// import awsecr adapter
_ "github.com/goharbor/harbor/src/pkg/reg/adapter/awsecr"
// import azurecr adapter
_ "github.com/goharbor/harbor/src/pkg/reg/adapter/azurecr"
// import dockerhub adapter
_ "github.com/goharbor/harbor/src/pkg/reg/adapter/dockerhub"
// import dtr adapter
_ "github.com/goharbor/harbor/src/pkg/reg/adapter/dtr"
// import githubcr adapter
_ "github.com/goharbor/harbor/src/pkg/reg/adapter/githubcr"
// import gitlab adapter
_ "github.com/goharbor/harbor/src/pkg/reg/adapter/gitlab"
// import googlegcr adapter
_ "github.com/goharbor/harbor/src/pkg/reg/adapter/googlegcr"
// import harbor adapter
_ "github.com/goharbor/harbor/src/pkg/reg/adapter/harbor"
// import huawei adapter
_ "github.com/goharbor/harbor/src/pkg/reg/adapter/huawei"
// import jfrog adapter
_ "github.com/goharbor/harbor/src/pkg/reg/adapter/jfrog"
// import native adapter
_ "github.com/goharbor/harbor/src/pkg/reg/adapter/native"
// import quay adapter
_ "github.com/goharbor/harbor/src/pkg/reg/adapter/quay"
// import tencentcr adapter
_ "github.com/goharbor/harbor/src/pkg/reg/adapter/tencentcr"
"github.com/goharbor/harbor/src/pkg/reg/model"
)

15
src/jobservice/job/impl/replication/replication_test.go
View File

@ -49,21 +49,6 @@ func TestParseParam(t *testing.T) {
assert.Equal(t, "tom", p.Name)
}
func TestParseParams(t *testing.T) {
params := map[string]interface{}{
"src_resource": `{"type":"chart"}`,
"dst_resource": `{"type":"chart"}`,
"speed": 1024,
"copy_by_chunk": true,
}
res, dst, opts, err := parseParams(params)
require.Nil(t, err)
assert.Equal(t, "chart", string(res.Type))
assert.Equal(t, "chart", string(dst.Type))
assert.Equal(t, int32(1024), opts.Speed)
assert.True(t, opts.CopyByChunk)
}
func TestMaxFails(t *testing.T) {
rep := &Replication{}
assert.Equal(t, uint(3), rep.MaxFails())

3
src/lib/config/metadata/metadata_test.go
View File

@ -24,12 +24,11 @@ func TestCfgMetaData_InitFromArray(t *testing.T) {
testArray := []Item{
{Scope: SystemScope, Group: BasicGroup, EnvKey: "HARBOR_ADMIN_PASSWORD", DefaultValue: "", Name: common.AdminInitialPassword, ItemType: &PasswordType{}, Editable: true},
{Scope: UserScope, Group: BasicGroup, EnvKey: "AUTH_MODE", DefaultValue: "db_auth", Name: common.AUTHMode, ItemType: &StringType{}, Editable: false},
{Scope: SystemScope, Group: BasicGroup, EnvKey: "CHART_REPOSITORY_URL", DefaultValue: "http://chartmuseum:9999", Name: common.ChartRepoURL, ItemType: &StringType{}, Editable: false},
}
curInst := Instance()
curInst.initFromArray(testArray)
if len(metaDataInstance.metaMap) != 3 {
if len(metaDataInstance.metaMap) != 2 {
t.Errorf("Can not initial metadata, size %v", len(metaDataInstance.metaMap))
}
item, ok := curInst.GetByName(common.AdminInitialPassword)

3
src/lib/config/metadata/metadatalist.go
View File

@ -65,8 +65,8 @@ var (
{Name: common.AdminInitialPassword, Scope: SystemScope, Group: BasicGroup, EnvKey: "HARBOR_ADMIN_PASSWORD", DefaultValue: "", ItemType: &PasswordType{}, Editable: true},
{Name: common.AUTHMode, Scope: UserScope, Group: BasicGroup, EnvKey: "AUTH_MODE", DefaultValue: "db_auth", ItemType: &AuthModeType{}, Editable: false, Description: `The auth mode of current system, such as "db_auth", "ldap_auth", "oidc_auth"`},
{Name: common.PrimaryAuthMode, Scope: UserScope, Group: BasicGroup, EnvKey: "PRIMARY_AUTH_MODE", DefaultValue: "false", ItemType: &BoolType{}, Description: `Use current auth mode as a primary one`},
{Name: common.ChartRepoURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "CHART_REPOSITORY_URL", DefaultValue: "http://chartmuseum:9999", ItemType: &StringType{}, Editable: false},
{Name: common.TrivyAdapterURL, Scope: SystemScope, Group: TrivyGroup, EnvKey: "TRIVY_ADAPTER_URL", DefaultValue: "http://trivy-adapter:8080", ItemType: &StringType{}, Editable: false},
@ -144,7 +144,6 @@ var (
{Name: common.OIDCAutoOnboard, Scope: UserScope, Group: OIDCGroup, DefaultValue: "false", ItemType: &BoolType{}, Description: `Auto onboard the OIDC user`},
{Name: common.OIDCExtraRedirectParms, Scope: UserScope, Group: OIDCGroup, DefaultValue: "{}", ItemType: &StringToStringMapType{}, Description: `Extra parameters to add when redirect request to OIDC provider`},
{Name: common.WithChartMuseum, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_CHARTMUSEUM", DefaultValue: "false", ItemType: &BoolType{}, Editable: true},
{Name: common.WithTrivy, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_TRIVY", DefaultValue: "false", ItemType: &BoolType{}, Editable: true},
{Name: common.WithNotary, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_NOTARY", DefaultValue: "false", ItemType: &BoolType{}, Editable: true},
// the unit of expiration is days

16
src/lib/config/systemconfig.go
View File

@ -30,7 +30,6 @@ package config
import (
"context"
"errors"
"os"
"strconv"
"strings"
@ -152,21 +151,6 @@ func WithTrivy() bool {
return DefaultMgr().Get(backgroundCtx, common.WithTrivy).GetBool()
}
// WithChartMuseum returns a bool to indicate if chartmuseum is deployed with Harbor.
func WithChartMuseum() bool {
return DefaultMgr().Get(backgroundCtx, common.WithChartMuseum).GetBool()
}
// GetChartMuseumEndpoint returns the endpoint of the chartmuseum service
// otherwise an non nil error is returned
func GetChartMuseumEndpoint() (string, error) {
chartEndpoint := strings.TrimSpace(DefaultMgr().Get(backgroundCtx, common.ChartRepoURL).GetString())
if len(chartEndpoint) == 0 {
return "", errors.New("empty chartmuseum endpoint")
}
return chartEndpoint, nil
}
// ExtEndpoint returns the external URL of Harbor: protocol://host:port
func ExtEndpoint() (string, error) {
return DefaultMgr().Get(backgroundCtx, common.ExtEndpoint).GetString(), nil

5
src/lib/config/test/userconfig_test.go
View File

@ -42,9 +42,8 @@ func TestConfig(t *testing.T) {
dao.PrepareTestData([]string{"delete from properties where k='scan_all_policy'"}, []string{})
defaultCACertPath = path.Join(currPath(), "test", "ca.crt")
c := map[string]interface{}{
common.WithTrivy: false,
common.WithChartMuseum: false,
common.WithNotary: false,
common.WithTrivy: false,
common.WithNotary: false,
}
Init()

4
src/lib/selector/candidate.go
View File

@ -26,8 +26,6 @@ import (
const (
// Image kind
Image = "image"
// Chart kind
Chart = "chart"
)
// Repository of candidate
@ -71,7 +69,7 @@ type Candidate struct {
// Repository name
Repository string `json:"repository"`
// Kind of the candidate
// "image" or "chart"
// "image"
Kind string `json:"kind"`
// Tags attached with the candidate
Tags []string `json:"tags"`

2
src/pkg/artifact/dao/model.go
View File

@ -30,7 +30,7 @@ func init() {
// Artifact model in database
type Artifact struct {
ID int64 `orm:"pk;auto;column(id)"`
Type string `orm:"column(type)"` // image or chart
Type string `orm:"column(type)"` // image, chart or other OCI compatible
MediaType string `orm:"column(media_type)"` // the media type of artifact
ManifestMediaType string `orm:"column(manifest_media_type)"` // the media type of manifest/index
ProjectID int64 `orm:"column(project_id)"` // needed for quota

2
src/pkg/artifact/model.go
View File

@ -31,7 +31,7 @@ import (
// for all users.
type Artifact struct {
ID int64 `json:"id"`
Type string `json:"type"` // image, chart, etc
Type string `json:"type"` // image, chart or other OCI compatible
MediaType string `json:"media_type"` // the media type of artifact. Mostly, it's the value of `manifest.config.mediatype`
ManifestMediaType string `json:"manifest_media_type"` // the media type of manifest/index
ProjectID int64 `json:"project_id"`

357
src/testing/chart_utility.go → src/pkg/chart/operator_test.go
View File

File diff suppressed because one or more lines are too long

28
src/pkg/chart/opetator_test.go
View File

@ -1,28 +0,0 @@
package chart
import (
"testing"
htesting "github.com/goharbor/harbor/src/testing"
)
func TestGetChartDetails(t *testing.T) {
chartOpr := NewOperator()
_, err := chartOpr.GetDetails(htesting.HelmChartContent)
if err != nil {
t.Fatal(err)
}
// ToDo add a v3 supported test data
// if len(chartDetails.Dependencies) == 0 {
// t.Fatal("At least 1 dependency exitsing, but we got 0 now")
// }
// if len(chartDetails.Values) == 0 {
// t.Fatal("At least 1 value existing, but we got 0 now")
// }
// if chartDetails.Values["adminserver.adminPassword"] != "Harbor12345" {
// t.Fatalf("The value of 'adminserver.adminPassword' should be 'Harbor12345' but we got '%s' now", chartDetails.Values["adminserver.adminPassword"])
// }
}

40
src/pkg/clients/core/chart.go
View File

@ -1,40 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package core
import (
"fmt"
"github.com/goharbor/harbor/src/chartserver"
)
func (c *client) ListAllCharts(project, repository string) ([]*chartserver.ChartVersion, error) {
url := c.buildURL(fmt.Sprintf("/api/chartrepo/%s/charts/%s", project, repository))
var charts []*chartserver.ChartVersion
if err := c.httpclient.Get(url, &charts); err != nil {
return nil, err
}
return charts, nil
}
func (c *client) DeleteChart(project, repository, version string) error {
url := c.buildURL(fmt.Sprintf("/api/chartrepo/%s/charts/%s/%s", project, repository, version))
return c.httpclient.Delete(url)
}
func (c *client) DeleteChartRepository(project, repository string) error {
url := c.buildURL(fmt.Sprintf("/api/chartrepo/%s/charts/%s", project, repository))
return c.httpclient.Delete(url)
}

9
src/pkg/clients/core/client.go
View File

@ -18,7 +18,6 @@ import (
"fmt"
"net/http"
"github.com/goharbor/harbor/src/chartserver"
chttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
modelsv2 "github.com/goharbor/harbor/src/controller/artifact"
@ -29,7 +28,6 @@ import (
// and we should expand it when needed
type Client interface {
ArtifactClient
ChartClient
}
// ArtifactClient defines the methods that an image client should implement
@ -39,13 +37,6 @@ type ArtifactClient interface {
DeleteArtifactRepository(project, repository string) error
}
// ChartClient defines the methods that a chart client should implement
type ChartClient interface {
ListAllCharts(project, repository string) ([]*chartserver.ChartVersion, error)
DeleteChart(project, repository, version string) error
DeleteChartRepository(project, repository string) error
}
// New returns an instance of the client which is a default implement for Client
func New(url string, httpclient *http.Client, modifiers ...modifier.Modifier) Client {
return &client{

8
src/pkg/notification/job/dao/dao_test.go
View File

@ -99,7 +99,7 @@ func (suite *DaoTestSuite) TestGet() {
id, err := suite.dao.Create(orm.Context(), &model.Job{
PolicyID: 2222,
EventType: "pushChart",
EventType: "pullImage",
NotifyType: "http",
Status: "pending",
JobDetail: "{\"type\":\"pushImage\",\"occur_at\":1563536782,\"event_data\":{\"resources\":[{\"digest\":\"sha256:bf1684a6e3676389ec861c602e97f27b03f14178e5bc3f70dce198f9f160cce9\",\"tag\":\"v1.0\",\"resource_url\":\"10.194.32.23/myproj/alpine:v1.0\"}],\"repository\":{\"date_created\":1563505587,\"name\":\"alpine\",\"namespace\":\"myproj\",\"repo_full_name\":\"myproj/alpine\",\"repo_type\":\"private\"}},\"operator\":\"admin\"}",
@ -109,7 +109,7 @@ func (suite *DaoTestSuite) TestGet() {
r, err := suite.dao.Get(orm.Context(), id)
suite.Nil(err)
suite.Equal("pushChart", r.EventType)
suite.Equal("pullImage", r.EventType)
}
func (suite *DaoTestSuite) TestUpdate() {
@ -163,7 +163,7 @@ func (suite *DaoTestSuite) TestDeleteByPolicyID() {
func (suite *DaoTestSuite) TestGetLastTriggerJobsGroupByEventType() {
_, err := suite.dao.Create(orm.Context(), &model.Job{
PolicyID: 3333,
EventType: "pushChart",
EventType: "replicateImage",
NotifyType: "http",
Status: "pending",
JobDetail: "{\"type\":\"pushImage\",\"occur_at\":1563536782,\"event_data\":{\"resources\":[{\"digest\":\"sha256:bf1684a6e3676389ec861c602e97f27b03f14178e5bc3f70dce198f9f160cce9\",\"tag\":\"v1.0\",\"resource_url\":\"10.194.32.23/myproj/alpine:v1.0\"}],\"repository\":{\"date_created\":1563505587,\"name\":\"alpine\",\"namespace\":\"myproj\",\"repo_full_name\":\"myproj/alpine\",\"repo_type\":\"private\"}},\"operator\":\"admin\"}",
@ -172,7 +172,7 @@ func (suite *DaoTestSuite) TestGetLastTriggerJobsGroupByEventType() {
suite.Nil(err)
_, err = suite.dao.Create(orm.Context(), &model.Job{
PolicyID: 3333,
EventType: "pullChart",
EventType: "pullImage",
NotifyType: "http",
Status: "pending",
JobDetail: "{\"type\":\"pushImage\",\"occur_at\":1563536782,\"event_data\":{\"resources\":[{\"digest\":\"sha256:bf1684a6e3676389ec861c602e97f27b03f14178e5bc3f70dce198f9f160cce9\",\"tag\":\"v1.0\",\"resource_url\":\"10.194.32.23/myproj/alpine:v1.0\"}],\"repository\":{\"date_created\":1563505587,\"name\":\"alpine\",\"namespace\":\"myproj\",\"repo_full_name\":\"myproj/alpine\",\"repo_type\":\"private\"}},\"operator\":\"admin\"}",

Some files were not shown because too many files have changed in this diff Show More