mirror of
https://github.com/goharbor/harbor.git
synced 2024-11-25 11:46:43 +01:00
Merge pull request #10694 from danielpacak/feature/install_with_trivy
chore(install): Add --with-trivy arg to the installation script
This commit is contained in:
commit
f1374737f6
17
Makefile
17
Makefile
@ -77,6 +77,7 @@ REGISTRYPROJECTNAME=goharbor
|
||||
DEVFLAG=true
|
||||
NOTARYFLAG=false
|
||||
CLAIRFLAG=false
|
||||
TRIVYFLAG=false
|
||||
HTTPPROXY=
|
||||
BUILDBIN=false
|
||||
MIGRATORFLAG=false
|
||||
@ -104,6 +105,8 @@ MIGRATORVERSION=$(VERSIONTAG)
|
||||
REDISVERSION=$(VERSIONTAG)
|
||||
NOTARYMIGRATEVERSION=v3.5.4
|
||||
CLAIRADAPTERVERSION=v1.0.1
|
||||
TRIVYVERSION=v0.4.3
|
||||
TRIVYADAPTERVERSION=v0.2.3
|
||||
|
||||
# version of chartmuseum
|
||||
CHARTMUSEUMVERSION=v0.9.0
|
||||
@ -117,6 +120,8 @@ REGISTRY_VERSION: $(REGISTRYVERSION)
|
||||
NOTARY_VERSION: $(NOTARYVERSION)
|
||||
CLAIR_VERSION: $(CLAIRVERSION)
|
||||
CLAIR_ADAPTER_VERSION: $(CLAIRADAPTERVERSION)
|
||||
TRIVY_VERSION: $(TRIVYVERSION)
|
||||
TRIVY_ADAPTER_VERSION: $(TRIVYADAPTERVERSION)
|
||||
CHARTMUSEUM_VERSION: $(CHARTMUSEUMVERSION)
|
||||
endef
|
||||
|
||||
@ -193,6 +198,9 @@ endif
|
||||
ifeq ($(CLAIRFLAG), true)
|
||||
PREPARECMD_PARA+= --with-clair
|
||||
endif
|
||||
ifeq ($(TRIVYFLAG), true)
|
||||
PREPARECMD_PARA+= --with-trivy
|
||||
endif
|
||||
# append chartmuseum parameters if set
|
||||
ifeq ($(CHARTFLAG), true)
|
||||
PREPARECMD_PARA+= --with-chartmuseum
|
||||
@ -276,6 +284,9 @@ endif
|
||||
ifeq ($(CLAIRFLAG), true)
|
||||
DOCKERSAVE_PARA+= goharbor/clair-photon:$(CLAIRVERSION)-$(VERSIONTAG) goharbor/clair-adapter-photon:$(CLAIRADAPTERVERSION)-$(VERSIONTAG)
|
||||
endif
|
||||
ifeq ($(TRIVYFLAG), true)
|
||||
DOCKERSAVE_PARA+= goharbor/trivy-adapter-photon:$(TRIVYADAPTERVERSION)-$(VERSIONTAG)
|
||||
endif
|
||||
ifeq ($(MIGRATORFLAG), true)
|
||||
DOCKERSAVE_PARA+= goharbor/harbor-migrator:$(MIGRATORVERSION)
|
||||
endif
|
||||
@ -348,14 +359,16 @@ prepare: update_prepare_version
|
||||
|
||||
build:
|
||||
make -f $(MAKEFILEPATH_PHOTON)/Makefile $(BUILDTARGET) -e DEVFLAG=$(DEVFLAG) -e GOBUILDIMAGE=$(GOBUILDIMAGE) \
|
||||
-e REGISTRYVERSION=$(REGISTRYVERSION) -e REGISTRY_SRC_TAG=$(REGISTRY_SRC_TAG) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
|
||||
-e REGISTRYVERSION=$(REGISTRYVERSION) -e REGISTRY_SRC_TAG=$(REGISTRY_SRC_TAG) -e NGINXVERSION=$(NGINXVERSION) \
|
||||
-e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
|
||||
-e TRIVYVERSION=$(TRIVYVERSION) -e TRIVYADAPTERVERSION=$(TRIVYADAPTERVERSION) \
|
||||
-e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRADAPTERVERSION=$(CLAIRADAPTERVERSION) -e VERSIONTAG=$(VERSIONTAG) \
|
||||
-e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \
|
||||
-e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER) \
|
||||
-e NPM_REGISTRY=$(NPM_REGISTRY) -e BASEIMAGETAG=$(BASEIMAGETAG)
|
||||
|
||||
build_base_docker:
|
||||
@for name in chartserver clair clair-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
|
||||
@for name in chartserver clair clair-adapter trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
|
||||
echo $$name ; \
|
||||
$(DOCKERBUILD) --pull -f $(MAKEFILEPATH_PHOTON)/$$name/Dockerfile.base -t goharbor/harbor-$$name-base:$(BASEIMAGETAG) . ; \
|
||||
$(PUSHSCRIPTPATH)/$(PUSHSCRIPTNAME) goharbor/harbor-$$name-base:$(BASEIMAGETAG) $(REGISTRYUSER) $(REGISTRYPASSWORD) ; \
|
||||
|
@ -141,6 +141,7 @@ _version: 1.10.0
|
||||
# jobservice_db_index: 2
|
||||
# chartmuseum_db_index: 3
|
||||
# clair_db_index: 4
|
||||
# trivy_db_index: 5
|
||||
|
||||
# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
|
||||
# uaa:
|
||||
|
@ -10,6 +10,7 @@ set +o noglob
|
||||
usage=$'Please set hostname and other necessary attributes in harbor.yml first. DO NOT use localhost or 127.0.0.1 for hostname, because Harbor needs to be accessed by external clients.
|
||||
Please set --with-notary if needs enable Notary in Harbor, and set ui_url_protocol/ssl_cert/ssl_cert_key in harbor.yml bacause notary must run under https.
|
||||
Please set --with-clair if needs enable Clair in Harbor
|
||||
Please set --with-tivy if needs enable Trivy in Harbor
|
||||
Please set --with-chartmuseum if needs enable Chartmuseum in Harbor'
|
||||
item=0
|
||||
|
||||
@ -17,6 +18,8 @@ item=0
|
||||
with_notary=$false
|
||||
# clair is not enabled by default
|
||||
with_clair=$false
|
||||
# trivy is not enabled by default
|
||||
with_trivy=$false
|
||||
# chartmuseum is not enabled by default
|
||||
with_chartmuseum=$false
|
||||
|
||||
@ -29,6 +32,8 @@ while [ $# -gt 0 ]; do
|
||||
with_notary=true;;
|
||||
--with-clair)
|
||||
with_clair=true;;
|
||||
--with-trivy)
|
||||
with_trivy=true;;
|
||||
--with-chartmuseum)
|
||||
with_chartmuseum=true;;
|
||||
*)
|
||||
@ -70,6 +75,10 @@ if [ $with_clair ]
|
||||
then
|
||||
prepare_para="${prepare_para} --with-clair"
|
||||
fi
|
||||
if [ $with_trivy ]
|
||||
then
|
||||
prepare_para="${prepare_para} --with-trivy"
|
||||
fi
|
||||
if [ $with_chartmuseum ]
|
||||
then
|
||||
prepare_para="${prepare_para} --with-chartmuseum"
|
||||
|
@ -60,11 +60,18 @@ DOCKERIMAGENAME_POSTGRESQL=goharbor/postgresql-photon
|
||||
DOCKERFILEPATH_CLAIR=$(DOCKERFILEPATH)/clair
|
||||
DOCKERFILENAME_CLAIR=Dockerfile
|
||||
DOCKERIMAGENAME_CLAIR=goharbor/clair-photon
|
||||
CLAIR_ADAPTER_DOWNLOAD_URL=https://github.com/goharbor/harbor-scanner-clair/releases/download/$(CLAIRADAPTERVERSION)/harbor-scanner-clair_$(CLAIRADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz
|
||||
|
||||
DOCKERFILEPATH_CLAIR_ADAPTER=$(DOCKERFILEPATH)/clair-adapter
|
||||
DOCKERFILENAME_CLAIR_ADAPTER=Dockerfile
|
||||
DOCKERIMAGENAME_CLAIR_ADAPTER=goharbor/clair-adapter-photon
|
||||
|
||||
DOCKERFILEPATH_TRIVY_ADAPTER=$(DOCKERFILEPATH)/trivy-adapter
|
||||
DOCKERFILENAME_TRIVY_ADAPTER=Dockerfile
|
||||
DOCKERIMAGENAME_TRIVY_ADAPTER=goharbor/trivy-adapter-photon
|
||||
TRIVY_DOWNLOAD_URL=https://github.com/aquasecurity/trivy/releases/download/$(TRIVYVERSION)/trivy_$(TRIVYVERSION:v%=%)_Linux-64bit.tar.gz
|
||||
TRIVY_ADAPTER_DOWNLOAD_URL=https://github.com/aquasecurity/harbor-scanner-trivy/releases/download/$(TRIVYADAPTERVERSION)/harbor-scanner-trivy_$(TRIVYADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz
|
||||
|
||||
DOCKERFILEPATH_NGINX=$(DOCKERFILEPATH)/nginx
|
||||
DOCKERFILENAME_NGINX=Dockerfile
|
||||
DOCKERIMAGENAME_NGINX=goharbor/nginx-photon
|
||||
@ -148,17 +155,40 @@ _build_clair_adapter:
|
||||
@if [ "$(CLAIRFLAG)" = "true" ] ; then \
|
||||
if [ "$(BUILDBIN)" != "true" ] ; then \
|
||||
rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary && mkdir -p $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary && \
|
||||
$(call _extract_archive, https://github.com/goharbor/harbor-scanner-clair/releases/download/$(CLAIRADAPTERVERSION)/harbor-scanner-clair_$(CLAIRADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz, $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/) && \
|
||||
$(call _extract_archive, $(CLAIR_ADAPTER_DOWNLOAD_URL), $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/) && \
|
||||
mv $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/scanner-clair $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/harbor-scanner-clair; \
|
||||
else \
|
||||
cd $(DOCKERFILEPATH_CLAIR_ADAPTER) && $(DOCKERFILEPATH_CLAIR_ADAPTER)/builder $(CLAIRADAPTERVERSION) && cd - ; \
|
||||
cd $(DOCKERFILEPATH_CLAIR_ADAPTER) && $(DOCKERFILEPATH_CLAIR_ADAPTER)/builder.sh $(CLAIRADAPTERVERSION) && cd - ; \
|
||||
fi ; \
|
||||
echo "building clair adapter container for photon..." ; \
|
||||
$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) -f $(DOCKERFILEPATH_CLAIR_ADAPTER)/$(DOCKERFILENAME_CLAIR_ADAPTER) -t $(DOCKERIMAGENAME_CLAIR_ADAPTER):$(CLAIRADAPTERVERSION)-$(VERSIONTAG) . ; \
|
||||
echo "Building Clair adapter container for photon..." ; \
|
||||
$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) \
|
||||
-f $(DOCKERFILEPATH_CLAIR_ADAPTER)/$(DOCKERFILENAME_CLAIR_ADAPTER) \
|
||||
-t $(DOCKERIMAGENAME_CLAIR_ADAPTER):$(CLAIRADAPTERVERSION)-$(VERSIONTAG) . ; \
|
||||
rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary; \
|
||||
echo "Done." ; \
|
||||
fi
|
||||
|
||||
_build_trivy_adapter:
|
||||
@if [ "$(TRIVYFLAG)" = "true" ] ; then \
|
||||
rm -rf $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary && mkdir -p $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary ; \
|
||||
echo "Downloading Trivy scanner $(TRIVYVERSION)..." ; \
|
||||
$(call _extract_archive, $(TRIVY_DOWNLOAD_URL), $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary/) ; \
|
||||
if [ "$(BUILDBIN)" != "true" ] ; then \
|
||||
echo "Downloading Trivy adapter $(TRIVYADAPTERVERSION)..." ; \
|
||||
$(call _extract_archive, $(TRIVY_ADAPTER_DOWNLOAD_URL), $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary/) ; \
|
||||
else \
|
||||
echo "Building Trivy adapter $(TRIVYADAPTERVERSION) from sources..." ; \
|
||||
cd $(DOCKERFILEPATH_TRIVY_ADAPTER) && $(DOCKERFILEPATH_TRIVY_ADAPTER)/builder.sh $(TRIVYADAPTERVERSION) && cd - ; \
|
||||
fi ; \
|
||||
echo "Building Trivy adapter container for photon..." ; \
|
||||
$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) \
|
||||
--build-arg trivy_version=$(TRIVYVERSION) \
|
||||
-f $(DOCKERFILEPATH_TRIVY_ADAPTER)/$(DOCKERFILENAME_TRIVY_ADAPTER) \
|
||||
-t $(DOCKERIMAGENAME_TRIVY_ADAPTER):$(TRIVYADAPTERVERSION)-$(VERSIONTAG) . ; \
|
||||
rm -rf $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary; \
|
||||
echo "Done." ; \
|
||||
fi
|
||||
|
||||
_build_chart_server:
|
||||
@if [ "$(CHARTFLAG)" = "true" ] ; then \
|
||||
if [ "$(BUILDBIN)" != "true" ] ; then \
|
||||
@ -231,7 +261,7 @@ define _get_binary
|
||||
$(WGET) --timeout 30 --no-check-certificate $1 -O $2 || exit 1
|
||||
endef
|
||||
|
||||
build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_clair_adapter _build_redis _build_migrator _build_chart_server
|
||||
build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_clair_adapter _build_trivy_adapter _build_redis _build_migrator _build_chart_server
|
||||
|
||||
cleanimage:
|
||||
@echo "cleaning image for photon..."
|
||||
|
@ -15,25 +15,25 @@ set -e
|
||||
mkdir -p binary
|
||||
rm -rf binary/harbor-scanner-clair || true
|
||||
|
||||
cd `dirname $0`
|
||||
cd $(dirname $0)
|
||||
cur=$PWD
|
||||
|
||||
# the temp folder to store distribution source code...
|
||||
TEMP=`mktemp -d ${TMPDIR-/tmp}/clair-adapter.XXXXXX`
|
||||
# The temporary directory to clone Clair adapter source code
|
||||
TEMP=$(mktemp -d ${TMPDIR-/tmp}/clair-adapter.XXXXXX)
|
||||
git clone https://github.com/goharbor/harbor-scanner-clair.git $TEMP
|
||||
cd $TEMP; git checkout $VERSION; cd -
|
||||
|
||||
echo 'build the clair adapter binary bases on the golang:1.13.4'
|
||||
echo "Building Clair adapter binary based on golang:1.13.4..."
|
||||
cp Dockerfile.binary $TEMP
|
||||
docker build -f $TEMP/Dockerfile.binary -t clair-adapter-golang $TEMP
|
||||
|
||||
echo 'copy the clair adapter binary to local...'
|
||||
echo "Copying Clair adapter binary from the container to the local directory..."
|
||||
ID=$(docker create clair-adapter-golang)
|
||||
docker cp $ID:/go/src/github.com/goharbor/harbor-scanner-clair/harbor-scanner-clair binary
|
||||
|
||||
docker rm -f $ID
|
||||
docker rmi -f clair-adapter-golang
|
||||
|
||||
echo "Build clair adapter binary success, then to build photon image..."
|
||||
echo "Building Clair adapter binary finished successfully"
|
||||
cd $cur
|
||||
rm -rf $TEMP
|
@ -15,6 +15,7 @@ from utils.notary import prepare_notary
|
||||
from utils.log import prepare_log_configs
|
||||
from utils.clair import prepare_clair
|
||||
from utils.clair_adapter import prepare_clair_adapter
|
||||
from utils.trivy_adapter import prepare_trivy_adapter
|
||||
from utils.chart import prepare_chartmuseum
|
||||
from utils.docker_compose import prepare_docker_compose
|
||||
from utils.nginx import prepare_nginx, nginx_confd_dir
|
||||
@ -27,11 +28,12 @@ old_private_key_pem_path, old_crt_path)
|
||||
@click.option('--conf', default=input_config_path, help="the path of Harbor configuration file")
|
||||
@click.option('--with-notary', is_flag=True, help="the Harbor instance is to be deployed with notary")
|
||||
@click.option('--with-clair', is_flag=True, help="the Harbor instance is to be deployed with clair")
|
||||
@click.option('--with-trivy', is_flag=True, help="the Harbor instance is to be deployed with Trivy")
|
||||
@click.option('--with-chartmuseum', is_flag=True, help="the Harbor instance is to be deployed with chart repository supporting")
|
||||
def main(conf, with_notary, with_clair, with_chartmuseum):
|
||||
def main(conf, with_notary, with_clair, with_trivy, with_chartmuseum):
|
||||
|
||||
delfile(config_dir)
|
||||
config_dict = parse_yaml_config(conf, with_notary=with_notary, with_clair=with_clair, with_chartmuseum=with_chartmuseum)
|
||||
config_dict = parse_yaml_config(conf, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
|
||||
try:
|
||||
validate(config_dict, notary_mode=with_notary)
|
||||
except Exception as e:
|
||||
@ -41,7 +43,7 @@ def main(conf, with_notary, with_clair, with_chartmuseum):
|
||||
|
||||
prepare_log_configs(config_dict)
|
||||
prepare_nginx(config_dict)
|
||||
prepare_core(config_dict, with_notary=with_notary, with_clair=with_clair, with_chartmuseum=with_chartmuseum)
|
||||
prepare_core(config_dict, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
|
||||
prepare_registry(config_dict)
|
||||
prepare_registry_ctl(config_dict)
|
||||
prepare_db(config_dict)
|
||||
@ -63,10 +65,13 @@ def main(conf, with_notary, with_clair, with_chartmuseum):
|
||||
prepare_clair(config_dict)
|
||||
prepare_clair_adapter(config_dict)
|
||||
|
||||
if with_trivy:
|
||||
prepare_trivy_adapter(config_dict)
|
||||
|
||||
if with_chartmuseum:
|
||||
prepare_chartmuseum(config_dict)
|
||||
|
||||
prepare_docker_compose(config_dict, with_clair, with_notary, with_chartmuseum)
|
||||
prepare_docker_compose(config_dict, with_clair, with_trivy, with_notary, with_chartmuseum)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
@ -26,6 +26,7 @@ CORE_SECRET={{core_secret}}
|
||||
JOBSERVICE_SECRET={{jobservice_secret}}
|
||||
WITH_NOTARY={{with_notary}}
|
||||
WITH_CLAIR={{with_clair}}
|
||||
WITH_TRIVY={{with_trivy}}
|
||||
CLAIR_DB_PASSWORD={{clair_db_password}}
|
||||
CLAIR_DB_HOST={{clair_db_host}}
|
||||
CLAIR_DB_PORT={{clair_db_port}}
|
||||
@ -37,6 +38,7 @@ CORE_LOCAL_URL={{core_local_url}}
|
||||
JOBSERVICE_URL={{jobservice_url}}
|
||||
CLAIR_URL={{clair_url}}
|
||||
CLAIR_ADAPTER_URL={{clair_adapter_url}}
|
||||
TRIVY_ADAPTER_URL={{trivy_adapter_url}}
|
||||
NOTARY_URL={{notary_url}}
|
||||
REGISTRY_STORAGE_PROVIDER_NAME={{storage_provider_name}}
|
||||
READ_ONLY=false
|
||||
|
@ -437,6 +437,35 @@ services:
|
||||
env_file:
|
||||
./common/config/clair-adapter/env
|
||||
{% endif %}
|
||||
{% if with_trivy %}
|
||||
trivy-adapter:
|
||||
container_name: trivy-adapter
|
||||
image: goharbor/trivy-adapter-photon:{{trivy_adapter_version}}
|
||||
restart: always
|
||||
cap_drop:
|
||||
- ALL
|
||||
dns_search: .
|
||||
networks:
|
||||
- harbor
|
||||
{% if external_redis == False %}
|
||||
depends_on:
|
||||
- redis
|
||||
{% endif %}
|
||||
volumes:
|
||||
- type: bind
|
||||
source: {{data_volume}}/trivy-adapter/trivy
|
||||
target: /home/scanner/.cache/trivy
|
||||
- type: bind
|
||||
source: {{data_volume}}/trivy-adapter/reports
|
||||
target: /home/scanner/.cache/reports
|
||||
logging:
|
||||
driver: "syslog"
|
||||
options:
|
||||
syslog-address: "tcp://127.0.0.1:1514"
|
||||
tag: "trivy-adapter"
|
||||
env_file:
|
||||
./common/config/trivy-adapter/env
|
||||
{% endif %}
|
||||
{% if with_chartmuseum %}
|
||||
chartmuseum:
|
||||
container_name: chartmuseum
|
||||
|
10
make/photon/prepare/templates/trivy-adapter/env.jinja
Normal file
10
make/photon/prepare/templates/trivy-adapter/env.jinja
Normal file
@ -0,0 +1,10 @@
|
||||
SCANNER_LOG_LEVEL={{log_level}}
|
||||
SCANNER_STORE_REDIS_URL={{redis_url_trivy}}
|
||||
SCANNER_STORE_REDIS_NAMESPACE=harbor.scanner.trivy:store
|
||||
SCANNER_JOB_QUEUE_REDIS_URL={{redis_url_trivy}}
|
||||
SCANNER_JOB_QUEUE_REDIS_NAMESPACE=harbor.scanner.trivy:job-queue
|
||||
SCANNER_TRIVY_CACHE_DIR=/home/scanner/.cache/trivy
|
||||
SCANNER_TRIVY_REPORTS_DIR=/home/scanner/.cache/reports
|
||||
SCANNER_TRIVY_VULN_TYPE=os,library
|
||||
SCANNER_TRIVY_SEVERITY=UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL
|
||||
SCANNER_TRIVY_IGNORE_UNFIXED=false
|
@ -94,7 +94,7 @@ def parse_versions():
|
||||
return versions
|
||||
|
||||
|
||||
def parse_yaml_config(config_file_path, with_notary, with_clair, with_chartmuseum):
|
||||
def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, with_chartmuseum):
|
||||
'''
|
||||
:param configs: config_parser object
|
||||
:returns: dict of configs
|
||||
@ -113,6 +113,7 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_chartmuseu
|
||||
'jobservice_url': 'http://jobservice:8080',
|
||||
'clair_url': 'http://clair:6060',
|
||||
'clair_adapter_url': 'http://clair-adapter:8080',
|
||||
'trivy_adapter_url': 'http://trivy-adapter:8080',
|
||||
'notary_url': 'http://notary-server:4443',
|
||||
'chart_repository_url': 'http://chartmuseum:9999'
|
||||
}
|
||||
@ -317,7 +318,7 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_chartmuseu
|
||||
config_dict['external_database'] = False
|
||||
|
||||
# update redis configs
|
||||
config_dict.update(get_redis_configs(configs.get("external_redis", None), with_clair))
|
||||
config_dict.update(get_redis_configs(configs.get("external_redis", None), with_clair, with_trivy))
|
||||
|
||||
# auto generated secret string for core
|
||||
config_dict['core_secret'] = generate_random_string(16)
|
||||
@ -351,7 +352,7 @@ def get_redis_url(db, redis=None):
|
||||
return "redis://{host}:{port}/{db}".format(**kwargs)
|
||||
|
||||
|
||||
def get_redis_configs(external_redis=None, with_clair=True):
|
||||
def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
|
||||
"""Returns configs for redis
|
||||
|
||||
>>> get_redis_configs()['external_redis']
|
||||
@ -362,6 +363,8 @@ def get_redis_configs(external_redis=None, with_clair=True):
|
||||
'redis://redis:6379/2'
|
||||
>>> get_redis_configs()['redis_url_clair']
|
||||
'redis://redis:6379/4'
|
||||
>>> get_redis_configs()['redis_url_trivy']
|
||||
'redis://redis:6379/5'
|
||||
|
||||
>>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['external_redis']
|
||||
True
|
||||
@ -371,9 +374,13 @@ def get_redis_configs(external_redis=None, with_clair=True):
|
||||
'redis://anonymous:pass@localhost:6379/2'
|
||||
>>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_clair']
|
||||
'redis://anonymous:pass@localhost:6379/4'
|
||||
>>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_trivy']
|
||||
'redis://anonymous:pass@localhost:6379/5'
|
||||
|
||||
>>> 'redis_url_clair' not in get_redis_configs(with_clair=False)
|
||||
True
|
||||
>>> 'redis_url_trivy' not in get_redis_configs(with_trivy=False)
|
||||
True
|
||||
"""
|
||||
|
||||
configs = dict(external_redis=bool(external_redis))
|
||||
@ -387,6 +394,7 @@ def get_redis_configs(external_redis=None, with_clair=True):
|
||||
'jobservice_db_index': 2,
|
||||
'chartmuseum_db_index': 3,
|
||||
'clair_db_index': 4,
|
||||
'trivy_db_index': 5,
|
||||
}
|
||||
|
||||
# overwriting existing keys by external_redis
|
||||
@ -406,4 +414,8 @@ def get_redis_configs(external_redis=None, with_clair=True):
|
||||
configs['redis_db_index_clair'] = redis['clair_db_index']
|
||||
configs['redis_url_clair'] = get_redis_url(configs['redis_db_index_clair'], redis)
|
||||
|
||||
if with_trivy:
|
||||
configs['redis_db_index_trivy'] = redis['trivy_db_index']
|
||||
configs['redis_url_trivy'] = get_redis_url(configs['redis_db_index_trivy'], redis)
|
||||
|
||||
return configs
|
||||
|
@ -13,7 +13,7 @@ core_conf = os.path.join(config_dir, "core", "app.conf")
|
||||
ca_download_dir = os.path.join(data_dir, 'ca_download')
|
||||
|
||||
|
||||
def prepare_core(config_dict, with_notary, with_clair, with_chartmuseum):
|
||||
def prepare_core(config_dict, with_notary, with_clair, with_trivy, with_chartmuseum):
|
||||
prepare_dir(ca_download_dir, uid=DEFAULT_UID, gid=DEFAULT_GID)
|
||||
prepare_dir(core_config_dir)
|
||||
# Render Core
|
||||
@ -30,6 +30,7 @@ def prepare_core(config_dict, with_notary, with_clair, with_chartmuseum):
|
||||
chart_cache_driver=chart_cache_driver,
|
||||
with_notary=with_notary,
|
||||
with_clair=with_clair,
|
||||
with_trivy=with_trivy,
|
||||
with_chartmuseum=with_chartmuseum,
|
||||
**config_dict)
|
||||
|
||||
@ -41,7 +42,6 @@ def prepare_core(config_dict, with_notary, with_clair, with_chartmuseum):
|
||||
xsrf_key=generate_random_string(40))
|
||||
|
||||
|
||||
|
||||
def copy_core_config(core_templates_path, core_config_path):
|
||||
shutil.copyfile(core_templates_path, core_config_path)
|
||||
print("Generated configuration file: %s" % core_config_path)
|
||||
|
@ -8,13 +8,14 @@ docker_compose_template_path = os.path.join(templates_dir, 'docker_compose', 'do
|
||||
docker_compose_yml_path = '/compose_location/docker-compose.yml'
|
||||
|
||||
# render docker-compose
|
||||
def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):
|
||||
def prepare_docker_compose(configs, with_clair, with_trivy, with_notary, with_chartmuseum):
|
||||
versions = parse_versions()
|
||||
VERSION_TAG = versions.get('VERSION_TAG') or 'dev'
|
||||
REGISTRY_VERSION = versions.get('REGISTRY_VERSION') or 'v2.7.1-patch-2819-2553'
|
||||
NOTARY_VERSION = versions.get('NOTARY_VERSION') or 'v0.6.1'
|
||||
CLAIR_VERSION = versions.get('CLAIR_VERSION') or 'v2.0.9'
|
||||
CLAIR_ADAPTER_VERSION = versions.get('CLAIR_ADAPTER_VERSION') or 'v1.0.0'
|
||||
TRIVY_ADAPTER_VERSION = versions.get('TRIVY_ADAPTER_VERSION') or 'v0.2.3'
|
||||
CHARTMUSEUM_VERSION = versions.get('CHARTMUSEUM_VERSION') or 'v0.9.0'
|
||||
|
||||
rendering_variables = {
|
||||
@ -24,6 +25,7 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):
|
||||
'notary_version': '{}-{}'.format(NOTARY_VERSION, VERSION_TAG),
|
||||
'clair_version': '{}-{}'.format(CLAIR_VERSION, VERSION_TAG),
|
||||
'clair_adapter_version': '{}-{}'.format(CLAIR_ADAPTER_VERSION, VERSION_TAG),
|
||||
'trivy_adapter_version': '{}-{}'.format(TRIVY_ADAPTER_VERSION, VERSION_TAG),
|
||||
'chartmuseum_version': '{}-{}'.format(CHARTMUSEUM_VERSION, VERSION_TAG),
|
||||
'data_volume': configs['data_volume'],
|
||||
'log_location': configs['log_location'],
|
||||
@ -34,6 +36,7 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):
|
||||
'external_database': configs['external_database'],
|
||||
'with_notary': with_notary,
|
||||
'with_clair': with_clair,
|
||||
'with_trivy': with_trivy,
|
||||
'with_chartmuseum': with_chartmuseum
|
||||
}
|
||||
|
||||
|
21
make/photon/prepare/utils/trivy_adapter.py
Normal file
21
make/photon/prepare/utils/trivy_adapter.py
Normal file
@ -0,0 +1,21 @@
|
||||
import os
|
||||
|
||||
from g import templates_dir, config_dir, data_dir, DEFAULT_UID, DEFAULT_GID
|
||||
from .jinja import render_jinja
|
||||
from .misc import prepare_dir
|
||||
|
||||
trivy_adapter_template_dir = os.path.join(templates_dir, "trivy-adapter")
|
||||
|
||||
|
||||
def prepare_trivy_adapter(config_dict):
|
||||
trivy_adapter_config_dir = prepare_dir(config_dir, "trivy-adapter")
|
||||
prepare_dir(data_dir, "trivy-adapter", "trivy", uid=DEFAULT_UID, gid=DEFAULT_GID)
|
||||
prepare_dir(data_dir, "trivy-adapter", "reports", uid=DEFAULT_UID, gid=DEFAULT_GID)
|
||||
|
||||
trivy_adapter_env_path = os.path.join(trivy_adapter_config_dir, "env")
|
||||
trivy_adapter_env_template = os.path.join(trivy_adapter_template_dir, "env.jinja")
|
||||
|
||||
render_jinja(
|
||||
trivy_adapter_env_template,
|
||||
trivy_adapter_env_path,
|
||||
**config_dict)
|
17
make/photon/trivy-adapter/Dockerfile
Normal file
17
make/photon/trivy-adapter/Dockerfile
Normal file
@ -0,0 +1,17 @@
|
||||
ARG harbor_base_image_version
|
||||
FROM goharbor/harbor-trivy-adapter-base:${harbor_base_image_version}
|
||||
|
||||
ARG trivy_version
|
||||
|
||||
COPY ./make/photon/trivy-adapter/binary/trivy /usr/local/bin/trivy
|
||||
COPY ./make/photon/trivy-adapter/binary/scanner-trivy /home/scanner/bin/scanner-trivy
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS 127.0.0.1:8080/probe/healthy || exit 1
|
||||
|
||||
ENV TRIVY_VERSION=${trivy_version}
|
||||
|
||||
USER scanner
|
||||
|
||||
ENTRYPOINT ["/home/scanner/bin/scanner-trivy"]
|
6
make/photon/trivy-adapter/Dockerfile.base
Normal file
6
make/photon/trivy-adapter/Dockerfile.base
Normal file
@ -0,0 +1,6 @@
|
||||
FROM photon:2.0
|
||||
|
||||
RUN tdnf install -y sudo rpm >> /dev/null \
|
||||
&& tdnf clean all \
|
||||
&& groupadd -r -g 10000 scanner \
|
||||
&& useradd --no-log-init -m -r -g 10000 -u 10000 scanner
|
7
make/photon/trivy-adapter/Dockerfile.binary
Normal file
7
make/photon/trivy-adapter/Dockerfile.binary
Normal file
@ -0,0 +1,7 @@
|
||||
FROM golang:1.13.4
|
||||
|
||||
ADD . /go/src/github.com/aquasecurity/harbor-scanner-trivy/
|
||||
WORKDIR /go/src/github.com/aquasecurity/harbor-scanner-trivy/
|
||||
|
||||
RUN export GOOS=linux GO111MODULE=on CGO_ENABLED=0 && \
|
||||
go build -o scanner-trivy cmd/scanner-trivy/main.go
|
35
make/photon/trivy-adapter/builder.sh
Executable file
35
make/photon/trivy-adapter/builder.sh
Executable file
@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
|
||||
set +e
|
||||
|
||||
if [ -z $1 ]; then
|
||||
error "Please set the 'version' variable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
VERSION="$1"
|
||||
|
||||
set -e
|
||||
|
||||
cd $(dirname $0)
|
||||
cur=$PWD
|
||||
|
||||
# The temporary directory to clone Trivy adapter source code
|
||||
TEMP=$(mktemp -d ${TMPDIR-/tmp}/trivy-adapter.XXXXXX)
|
||||
git clone https://github.com/aquasecurity/harbor-scanner-trivy.git $TEMP
|
||||
cd $TEMP; git checkout $VERSION; cd -
|
||||
|
||||
echo "Building Trivy adapter binary based on golang:1.13.4..."
|
||||
cp Dockerfile.binary $TEMP
|
||||
docker build -f $TEMP/Dockerfile.binary -t trivy-adapter-golang $TEMP
|
||||
|
||||
echo "Copying Trivy adapter binary from the container to the local directory..."
|
||||
ID=$(docker create trivy-adapter-golang)
|
||||
docker cp $ID:/go/src/github.com/aquasecurity/harbor-scanner-trivy/scanner-trivy binary
|
||||
|
||||
docker rm -f $ID
|
||||
docker rmi -f trivy-adapter-golang
|
||||
|
||||
echo "Building Trivy adapter binary finished successfully"
|
||||
cd $cur
|
||||
rm -rf $TEMP
|
@ -73,6 +73,7 @@ var (
|
||||
{Name: common.ClairDBUsername, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_DB_USERNAME", DefaultValue: "postgres", ItemType: &StringType{}, Editable: false},
|
||||
{Name: common.ClairURL, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_URL", DefaultValue: "http://clair:6060", ItemType: &StringType{}, Editable: false},
|
||||
{Name: common.ClairAdapterURL, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_ADAPTER_URL", DefaultValue: "http://clair-adapter:8080", ItemType: &StringType{}, Editable: false},
|
||||
{Name: common.TrivyAdapterURL, Scope: SystemScope, Group: ClairGroup, EnvKey: "TRIVY_ADAPTER_URL", DefaultValue: "http://trivy-adapter:8080", ItemType: &StringType{}, Editable: false},
|
||||
|
||||
{Name: common.CoreURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "CORE_URL", DefaultValue: "http://core:8080", ItemType: &StringType{}, Editable: false},
|
||||
{Name: common.CoreLocalURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "CORE_LOCAL_URL", DefaultValue: "http://127.0.0.1:8080", ItemType: &StringType{}, Editable: false},
|
||||
@ -150,6 +151,7 @@ var (
|
||||
|
||||
{Name: common.WithChartMuseum, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_CHARTMUSEUM", DefaultValue: "false", ItemType: &BoolType{}, Editable: true},
|
||||
{Name: common.WithClair, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_CLAIR", DefaultValue: "false", ItemType: &BoolType{}, Editable: true},
|
||||
{Name: common.WithTrivy, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_TRIVY", DefaultValue: "false", ItemType: &BoolType{}, Editable: true},
|
||||
{Name: common.WithNotary, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_NOTARY", DefaultValue: "false", ItemType: &BoolType{}, Editable: true},
|
||||
// the unit of expiration is minute, 43200 minutes = 30 days
|
||||
{Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "43200", ItemType: &IntType{}, Editable: true},
|
||||
|
@ -89,6 +89,7 @@ const (
|
||||
AdminInitialPassword = "admin_initial_password"
|
||||
WithNotary = "with_notary"
|
||||
WithClair = "with_clair"
|
||||
WithTrivy = "with_trivy"
|
||||
ScanAllPolicy = "scan_all_policy"
|
||||
ClairDBPassword = "clair_db_password"
|
||||
ClairDBHost = "clair_db_host"
|
||||
@ -123,6 +124,7 @@ const (
|
||||
ReadOnly = "read_only"
|
||||
ClairURL = "clair_url"
|
||||
ClairAdapterURL = "clair_adapter_url"
|
||||
TrivyAdapterURL = "trivy_adapter_url"
|
||||
NotaryURL = "notary_url"
|
||||
DefaultCoreEndpoint = "http://core:8080"
|
||||
DefaultNotaryEndpoint = "http://notary-server:4443"
|
||||
|
@ -348,6 +348,16 @@ func ClairAdapterEndpoint() string {
|
||||
return cfgMgr.Get(common.ClairAdapterURL).GetString()
|
||||
}
|
||||
|
||||
// WithTrivy returns a bool value to indicate if Harbor's deployed with Trivy.
|
||||
func WithTrivy() bool {
|
||||
return cfgMgr.Get(common.WithTrivy).GetBool()
|
||||
}
|
||||
|
||||
// TrivyAdapterURL returns the endpoint URL of a Trivy adapter instance, by default it's the one deployed within Harbor.
|
||||
func TrivyAdapterURL() string {
|
||||
return cfgMgr.Get(common.TrivyAdapterURL).GetString()
|
||||
}
|
||||
|
||||
// UAASettings returns the UAASettings to access UAA service.
|
||||
func UAASettings() (*models.UAASettings, error) {
|
||||
err := cfgMgr.Load()
|
||||
|
@ -208,31 +208,7 @@ func main() {
|
||||
log.Fatalf("Failed to initialize API handlers with error: %s", err.Error())
|
||||
}
|
||||
|
||||
if config.WithClair() {
|
||||
clairDB, err := config.ClairDB()
|
||||
if err != nil {
|
||||
log.Fatalf("failed to load clair database information: %v", err)
|
||||
}
|
||||
if err := dao.InitClairDB(clairDB); err != nil {
|
||||
log.Fatalf("failed to initialize clair database: %v", err)
|
||||
}
|
||||
|
||||
reg := &scanner.Registration{
|
||||
Name: "Clair",
|
||||
Description: "The clair scanner adapter",
|
||||
URL: config.ClairAdapterEndpoint(),
|
||||
UseInternalAddr: true,
|
||||
Immutable: true,
|
||||
}
|
||||
|
||||
if err := scan.EnsureScanner(reg, true); err != nil {
|
||||
log.Fatalf("failed to initialize clair scanner: %v", err)
|
||||
}
|
||||
} else {
|
||||
if err := scan.RemoveImmutableScanners(); err != nil {
|
||||
log.Warningf("failed to remove immutable scanners: %v", err)
|
||||
}
|
||||
}
|
||||
registerScanners()
|
||||
|
||||
closing := make(chan struct{})
|
||||
done := make(chan struct{})
|
||||
@ -291,3 +267,70 @@ func main() {
|
||||
|
||||
beego.RunWithMiddleWares("", middlewares.MiddleWares()...)
|
||||
}
|
||||
|
||||
func registerScanners() {
|
||||
wantedScanners := make([]scanner.Registration, 0)
|
||||
uninstallURLs := make([]string, 0)
|
||||
|
||||
if config.WithTrivy() {
|
||||
log.Info("Registering Trivy scanner")
|
||||
wantedScanners = append(wantedScanners, scanner.Registration{
|
||||
Name: "Trivy",
|
||||
Description: "The Trivy scanner adapter",
|
||||
URL: config.TrivyAdapterURL(),
|
||||
UseInternalAddr: true,
|
||||
Immutable: true,
|
||||
})
|
||||
} else {
|
||||
log.Info("Removing Trivy scanner")
|
||||
uninstallURLs = append(uninstallURLs, config.TrivyAdapterURL())
|
||||
}
|
||||
|
||||
if config.WithClair() {
|
||||
clairDB, err := config.ClairDB()
|
||||
if err != nil {
|
||||
log.Fatalf("failed to load clair database information: %v", err)
|
||||
}
|
||||
if err := dao.InitClairDB(clairDB); err != nil {
|
||||
log.Fatalf("failed to initialize clair database: %v", err)
|
||||
}
|
||||
|
||||
log.Info("Registering Clair scanner")
|
||||
wantedScanners = append(wantedScanners, scanner.Registration{
|
||||
Name: "Clair",
|
||||
Description: "The Clair scanner adapter",
|
||||
URL: config.ClairAdapterEndpoint(),
|
||||
UseInternalAddr: true,
|
||||
Immutable: true,
|
||||
})
|
||||
} else {
|
||||
log.Info("Removing Clair scanner")
|
||||
uninstallURLs = append(uninstallURLs, config.ClairAdapterEndpoint())
|
||||
}
|
||||
|
||||
if err := scan.EnsureScanners(wantedScanners); err != nil {
|
||||
log.Fatalf("failed to register scanners: %v", err)
|
||||
}
|
||||
|
||||
if defaultScannerURL := getDefaultScannerURL(); defaultScannerURL != "" {
|
||||
log.Infof("Setting %s as default scanner", defaultScannerURL)
|
||||
if err := scan.EnsureDefaultScanner(defaultScannerURL); err != nil {
|
||||
log.Fatalf("failed to set default scanner: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := scan.RemoveImmutableScanners(uninstallURLs); err != nil {
|
||||
log.Warningf("failed to remove scanners: %v", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func getDefaultScannerURL() string {
|
||||
if config.WithTrivy() {
|
||||
return config.TrivyAdapterURL()
|
||||
}
|
||||
if config.WithClair() {
|
||||
return config.ClairAdapterEndpoint()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
@ -15,8 +15,6 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/utils/log"
|
||||
"github.com/goharbor/harbor/src/pkg/q"
|
||||
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
|
||||
@ -30,36 +28,73 @@ var (
|
||||
scannerManager = sc.New()
|
||||
)
|
||||
|
||||
// EnsureScanner ensure the scanner which specially endpoint exists in the system
|
||||
func EnsureScanner(registration *scanner.Registration, resolveConflicts ...bool) error {
|
||||
q := &q.Query{
|
||||
Keywords: map[string]interface{}{"url": registration.URL},
|
||||
// EnsureScanners ensures that the scanners with the specified endpoints URLs exist in the system.
|
||||
func EnsureScanners(wantedScanners []scanner.Registration) (err error) {
|
||||
if len(wantedScanners) == 0 {
|
||||
return
|
||||
}
|
||||
endpointURLs := make([]string, len(wantedScanners))
|
||||
for i, ws := range wantedScanners {
|
||||
endpointURLs[i] = ws.URL
|
||||
}
|
||||
|
||||
// Check if the registration with the url already existing.
|
||||
registrations, err := scannerManager.List(q)
|
||||
list, err := scannerManager.List(&q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"ex_url__in": endpointURLs,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Errorf("listing scanners: %v", err)
|
||||
}
|
||||
existingScanners := make(map[string]*scanner.Registration)
|
||||
for _, li := range list {
|
||||
existingScanners[li.URL] = li
|
||||
}
|
||||
|
||||
if len(registrations) > 0 {
|
||||
return nil
|
||||
for _, ws := range wantedScanners {
|
||||
if _, exists := existingScanners[ws.URL]; exists {
|
||||
log.Infof("Scanner registration already exists: %s", ws.URL)
|
||||
continue
|
||||
}
|
||||
|
||||
var resolveConflict bool
|
||||
if len(resolveConflicts) > 0 {
|
||||
resolveConflict = resolveConflicts[0]
|
||||
}
|
||||
|
||||
var defaultReg *scanner.Registration
|
||||
defaultReg, err = scannerManager.GetDefault()
|
||||
err = createRegistration(&ws, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get the default scanner, error: %v", err)
|
||||
return errors.Errorf("creating registration: %s: %v", ws.URL, err)
|
||||
}
|
||||
log.Infof("Successfully registered %s scanner at %s", ws.Name, ws.URL)
|
||||
}
|
||||
|
||||
// Set the registration to be default one when no default registration exist in the system
|
||||
registration.IsDefault = defaultReg == nil
|
||||
return
|
||||
}
|
||||
|
||||
// EnsureDefaultScanner ensures that the scanner with the specified URL is set as default in the system.
|
||||
func EnsureDefaultScanner(scannerURL string) (err error) {
|
||||
defaultScanner, err := scannerManager.GetDefault()
|
||||
if err != nil {
|
||||
err = errors.Errorf("getting default scanner: %v", err)
|
||||
return
|
||||
}
|
||||
if defaultScanner != nil && defaultScanner.URL == scannerURL {
|
||||
log.Infof("The default scanner is already set: %s", defaultScanner.URL)
|
||||
return
|
||||
}
|
||||
scanners, err := scannerManager.List(&q.Query{
|
||||
Keywords: map[string]interface{}{"url": scannerURL},
|
||||
})
|
||||
if err != nil {
|
||||
err = errors.Errorf("listing scanners: %v", err)
|
||||
return
|
||||
}
|
||||
if len(scanners) != 1 {
|
||||
return errors.Errorf("expected only one scanner with URL %v but got %d", scannerURL, len(scanners))
|
||||
}
|
||||
err = scannerManager.SetAsDefault(scanners[0].UUID)
|
||||
if err != nil {
|
||||
err = errors.Errorf("setting %s as default scanner: %v", scannerURL, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func createRegistration(registration *scanner.Registration, resolveConflict bool) (err error) {
|
||||
for {
|
||||
_, err = scannerManager.Create(registration)
|
||||
if err != nil {
|
||||
@ -78,28 +113,30 @@ func EnsureScanner(registration *scanner.Registration, resolveConflicts ...bool)
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
log.Infof("initialized scanner named %s", registration.Name)
|
||||
return
|
||||
}
|
||||
|
||||
return err
|
||||
// RemoveImmutableScanners removes immutable scanner Registrations with the specified endpoint URLs.
|
||||
func RemoveImmutableScanners(urls []string) error {
|
||||
if len(urls) == 0 {
|
||||
return nil
|
||||
}
|
||||
query := &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"immutable": true,
|
||||
"ex_url__in": urls,
|
||||
},
|
||||
}
|
||||
|
||||
// RemoveImmutableScanners remove all immutable scanners in the system
|
||||
func RemoveImmutableScanners() error {
|
||||
q := &q.Query{
|
||||
Keywords: map[string]interface{}{"immutable": true},
|
||||
}
|
||||
|
||||
registrations, err := scannerManager.List(q)
|
||||
// TODO Instead of executing 1 to N SQL queries we might want to delete multiple rows with scannerManager.DeleteByImmutableAndURLIn(true, []string{})
|
||||
registrations, err := scannerManager.List(query)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Errorf("listing scanners: %v", err)
|
||||
}
|
||||
|
||||
for _, reg := range registrations {
|
||||
if err := scannerManager.Delete(reg.UUID); err != nil {
|
||||
return err
|
||||
return errors.Errorf("deleting scanner: %s: %v", reg.UUID, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -15,201 +15,260 @@
|
||||
package scan
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/pkg/q"
|
||||
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
|
||||
sc "github.com/goharbor/harbor/src/pkg/scan/scanner"
|
||||
"github.com/goharbor/harbor/src/pkg/scan/scanner/mocks"
|
||||
"github.com/goharbor/harbor/src/pkg/types"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type managerOptions struct {
|
||||
registrations []*scanner.Registration
|
||||
listError error
|
||||
getError error
|
||||
getDefaultError error
|
||||
createError error
|
||||
createErrorFn func(*scanner.Registration) error
|
||||
}
|
||||
func TestEnsureScanners(t *testing.T) {
|
||||
|
||||
func newManager(opts *managerOptions) sc.Manager {
|
||||
if opts == nil {
|
||||
opts = &managerOptions{}
|
||||
}
|
||||
|
||||
data := map[string]*scanner.Registration{}
|
||||
for _, reg := range opts.registrations {
|
||||
data[reg.URL] = reg
|
||||
}
|
||||
t.Run("Should do nothing when list of wanted scanners is empty", func(t *testing.T) {
|
||||
err := EnsureScanners([]scanner.Registration{})
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("Should return error when listing scanners fails", func(t *testing.T) {
|
||||
mgr := &mocks.Manager{}
|
||||
scannerManager = mgr
|
||||
|
||||
mgr.On("List", &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"ex_url__in": []string{"http://scanner:8080"},
|
||||
},
|
||||
}).Return(nil, errors.New("DB error"))
|
||||
|
||||
err := EnsureScanners([]scanner.Registration{
|
||||
{URL: "http://scanner:8080"},
|
||||
})
|
||||
|
||||
assert.EqualError(t, err, "listing scanners: DB error")
|
||||
mgr.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("Should create only non-existing scanners", func(t *testing.T) {
|
||||
mgr := &mocks.Manager{}
|
||||
scannerManager = mgr
|
||||
|
||||
mgr.On("List", &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"ex_url__in": []string{
|
||||
"http://trivy:8080",
|
||||
"http://clair:8080",
|
||||
},
|
||||
},
|
||||
}).Return([]*scanner.Registration{
|
||||
{URL: "http://clair:8080"},
|
||||
}, nil)
|
||||
mgr.On("Create", &scanner.Registration{
|
||||
URL: "http://trivy:8080",
|
||||
}).Return("uuid-trivy", nil)
|
||||
|
||||
err := EnsureScanners([]scanner.Registration{
|
||||
{URL: "http://trivy:8080"},
|
||||
{URL: "http://clair:8080"},
|
||||
})
|
||||
|
||||
assert.NoError(t, err)
|
||||
mgr.AssertExpectations(t)
|
||||
})
|
||||
|
||||
listFn := func(query *q.Query) []*scanner.Registration {
|
||||
if opts.listError != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
url := query.Keywords["url"]
|
||||
func TestEnsureDefaultScanner(t *testing.T) {
|
||||
|
||||
t.Run("Should return error when getting default scanner fails", func(t *testing.T) {
|
||||
mgr := &mocks.Manager{}
|
||||
scannerManager = mgr
|
||||
|
||||
mgr.On("GetDefault").Return(nil, errors.New("DB error"))
|
||||
|
||||
err := EnsureDefaultScanner("http://trivy:8080")
|
||||
assert.EqualError(t, err, "getting default scanner: DB error")
|
||||
mgr.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("Should do nothing when the default scanner is already set", func(t *testing.T) {
|
||||
mgr := &mocks.Manager{}
|
||||
scannerManager = mgr
|
||||
|
||||
mgr.On("GetDefault").Return(&scanner.Registration{
|
||||
URL: "http://trivy:8080",
|
||||
}, nil)
|
||||
|
||||
err := EnsureDefaultScanner("http://trivy:8080")
|
||||
assert.NoError(t, err)
|
||||
mgr.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("Should return error when listing scanners fails", func(t *testing.T) {
|
||||
mgr := &mocks.Manager{}
|
||||
scannerManager = mgr
|
||||
|
||||
mgr.On("GetDefault").Return(nil, nil)
|
||||
mgr.On("List", &q.Query{
|
||||
Keywords: map[string]interface{}{"url": "http://trivy:8080"},
|
||||
}).Return(nil, errors.New("DB error"))
|
||||
|
||||
err := EnsureDefaultScanner("http://trivy:8080")
|
||||
assert.EqualError(t, err, "listing scanners: DB error")
|
||||
mgr.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("Should return error when listing scanners returns unexpected scanners count", func(t *testing.T) {
|
||||
mgr := &mocks.Manager{}
|
||||
scannerManager = mgr
|
||||
|
||||
mgr.On("GetDefault").Return(nil, nil)
|
||||
mgr.On("List", &q.Query{
|
||||
Keywords: map[string]interface{}{"url": "http://trivy:8080"},
|
||||
}).Return([]*scanner.Registration{
|
||||
{URL: "http://trivy:8080"},
|
||||
{URL: "http://trivy:8080"},
|
||||
}, nil)
|
||||
|
||||
err := EnsureDefaultScanner("http://trivy:8080")
|
||||
assert.EqualError(t, err, "expected only one scanner with URL http://trivy:8080 but got 2")
|
||||
mgr.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("Should set the default scanner when it is not set", func(t *testing.T) {
|
||||
mgr := &mocks.Manager{}
|
||||
scannerManager = mgr
|
||||
|
||||
mgr.On("GetDefault").Return(nil, nil)
|
||||
mgr.On("List", &q.Query{
|
||||
Keywords: map[string]interface{}{"url": "http://trivy:8080"},
|
||||
}).Return([]*scanner.Registration{
|
||||
{
|
||||
UUID: "trivy-uuid",
|
||||
URL: "http://trivy:8080",
|
||||
},
|
||||
}, nil)
|
||||
mgr.On("SetAsDefault", "trivy-uuid").Return(nil)
|
||||
|
||||
err := EnsureDefaultScanner("http://trivy:8080")
|
||||
assert.NoError(t, err)
|
||||
mgr.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("Should return error when setting the default scanner fails", func(t *testing.T) {
|
||||
mgr := &mocks.Manager{}
|
||||
scannerManager = mgr
|
||||
|
||||
mgr.On("GetDefault").Return(nil, nil)
|
||||
mgr.On("List", &q.Query{
|
||||
Keywords: map[string]interface{}{"url": "http://trivy:8080"},
|
||||
}).Return([]*scanner.Registration{
|
||||
{
|
||||
UUID: "trivy-uuid",
|
||||
URL: "http://trivy:8080",
|
||||
},
|
||||
}, nil)
|
||||
mgr.On("SetAsDefault", "trivy-uuid").Return(errors.New("DB error"))
|
||||
|
||||
err := EnsureDefaultScanner("http://trivy:8080")
|
||||
assert.EqualError(t, err, "setting http://trivy:8080 as default scanner: DB error")
|
||||
mgr.AssertExpectations(t)
|
||||
})
|
||||
|
||||
var results []*scanner.Registration
|
||||
for key, reg := range data {
|
||||
if url == key {
|
||||
results = append(results, reg)
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
func TestRemoveImmutableScanners(t *testing.T) {
|
||||
|
||||
getFn := func(url string) *scanner.Registration {
|
||||
if opts.getError != nil {
|
||||
return nil
|
||||
}
|
||||
t.Run("Should do nothing when list of URLs is empty", func(t *testing.T) {
|
||||
mgr := &mocks.Manager{}
|
||||
scannerManager = mgr
|
||||
|
||||
return data[url]
|
||||
}
|
||||
err := RemoveImmutableScanners([]string{})
|
||||
assert.NoError(t, err)
|
||||
mgr.AssertExpectations(t)
|
||||
})
|
||||
|
||||
getDefaultFn := func() *scanner.Registration {
|
||||
if opts.getDefaultError != nil {
|
||||
return nil
|
||||
}
|
||||
t.Run("Should return error when listing scanners fails", func(t *testing.T) {
|
||||
mgr := &mocks.Manager{}
|
||||
scannerManager = mgr
|
||||
|
||||
for _, reg := range data {
|
||||
if reg.IsDefault {
|
||||
return reg
|
||||
}
|
||||
}
|
||||
mgr.On("List", &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"immutable": true,
|
||||
"ex_url__in": []string{"http://scanner:8080"},
|
||||
},
|
||||
}).Return(nil, errors.New("DB error"))
|
||||
|
||||
return nil
|
||||
}
|
||||
err := RemoveImmutableScanners([]string{"http://scanner:8080"})
|
||||
assert.EqualError(t, err, "listing scanners: DB error")
|
||||
mgr.AssertExpectations(t)
|
||||
})
|
||||
|
||||
createFn := func(reg *scanner.Registration) string {
|
||||
if opts.createError != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
data[reg.URL] = reg
|
||||
|
||||
return reg.URL
|
||||
}
|
||||
|
||||
createError := func(reg *scanner.Registration) error {
|
||||
if opts.createErrorFn != nil {
|
||||
return opts.createErrorFn(reg)
|
||||
}
|
||||
|
||||
return opts.createError
|
||||
}
|
||||
|
||||
mgr.On("List", mock.AnythingOfType("*q.Query")).Return(listFn, opts.listError)
|
||||
mgr.On("Get", mock.AnythingOfType("string")).Return(getFn, opts.getError)
|
||||
mgr.On("GetDefault").Return(getDefaultFn, opts.getDefaultError)
|
||||
mgr.On("Create", mock.AnythingOfType("*scanner.Registration")).Return(createFn, createError)
|
||||
|
||||
return mgr
|
||||
}
|
||||
|
||||
func TestEnsureScanner(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
t.Run("Should delete multiple scanners", func(t *testing.T) {
|
||||
mgr := &mocks.Manager{}
|
||||
scannerManager = mgr
|
||||
|
||||
registrations := []*scanner.Registration{
|
||||
{URL: "reg1"},
|
||||
}
|
||||
|
||||
// registration with the url exist in the system
|
||||
scannerManager = newManager(
|
||||
&managerOptions{
|
||||
registrations: registrations,
|
||||
{
|
||||
UUID: "uuid-1",
|
||||
URL: "http://scanner-1",
|
||||
},
|
||||
)
|
||||
assert.Nil(EnsureScanner(&scanner.Registration{URL: "reg1"}))
|
||||
{
|
||||
UUID: "uuid-2",
|
||||
URL: "http://scanner-2",
|
||||
}}
|
||||
|
||||
// list registrations got error
|
||||
scannerManager = newManager(
|
||||
&managerOptions{
|
||||
listError: errors.New("list registrations internal error"),
|
||||
},
|
||||
)
|
||||
assert.Error(EnsureScanner(&scanner.Registration{URL: "reg1"}))
|
||||
|
||||
// create registration got error
|
||||
scannerManager = newManager(
|
||||
&managerOptions{
|
||||
createError: errors.New("create registration internal error"),
|
||||
},
|
||||
)
|
||||
assert.Error(EnsureScanner(&scanner.Registration{URL: "reg1"}))
|
||||
|
||||
// get default registration got error
|
||||
scannerManager = newManager(
|
||||
&managerOptions{
|
||||
getDefaultError: errors.New("get default registration internal error"),
|
||||
},
|
||||
)
|
||||
assert.Error(EnsureScanner(&scanner.Registration{URL: "reg1"}))
|
||||
|
||||
// create registration when no registrations in the system
|
||||
scannerManager = newManager(nil)
|
||||
assert.Nil(EnsureScanner(&scanner.Registration{URL: "reg1"}))
|
||||
reg1, err := scannerManager.Get("reg1")
|
||||
assert.Nil(err)
|
||||
assert.NotNil(reg1)
|
||||
assert.True(reg1.IsDefault)
|
||||
|
||||
// create registration when there are registrations in the system
|
||||
scannerManager = newManager(
|
||||
&managerOptions{
|
||||
registrations: registrations,
|
||||
},
|
||||
)
|
||||
assert.Nil(EnsureScanner(&scanner.Registration{URL: "reg2"}))
|
||||
reg2, err := scannerManager.Get("reg2")
|
||||
assert.Nil(err)
|
||||
assert.NotNil(reg2)
|
||||
assert.True(reg2.IsDefault)
|
||||
|
||||
// create registration when there are registrations in the system and the default registration exist
|
||||
scannerManager = newManager(
|
||||
&managerOptions{
|
||||
registrations: []*scanner.Registration{
|
||||
{URL: "reg1", IsDefault: true},
|
||||
mgr.On("List", &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"immutable": true,
|
||||
"ex_url__in": []string{
|
||||
"http://scanner-1",
|
||||
"http://scanner-2",
|
||||
},
|
||||
},
|
||||
)
|
||||
assert.Nil(EnsureScanner(&scanner.Registration{URL: "reg3"}))
|
||||
reg3, err := scannerManager.Get("reg3")
|
||||
assert.Nil(err)
|
||||
assert.NotNil(reg3)
|
||||
assert.False(reg3.IsDefault)
|
||||
}
|
||||
}).Return(registrations, nil)
|
||||
mgr.On("Delete", "uuid-1").Return(nil)
|
||||
mgr.On("Delete", "uuid-2").Return(nil)
|
||||
|
||||
func TestEnsureScannerWithResolveConflict(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
err := RemoveImmutableScanners([]string{
|
||||
"http://scanner-1",
|
||||
"http://scanner-2",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
mgr.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("Should return error when deleting any scanner fails", func(t *testing.T) {
|
||||
mgr := &mocks.Manager{}
|
||||
scannerManager = mgr
|
||||
|
||||
registrations := []*scanner.Registration{
|
||||
{URL: "reg1"},
|
||||
}
|
||||
{
|
||||
UUID: "uuid-1",
|
||||
URL: "http://scanner-1",
|
||||
},
|
||||
{
|
||||
UUID: "uuid-2",
|
||||
URL: "http://scanner-2",
|
||||
}}
|
||||
|
||||
// create registration got ErrDupRows when its name is Clair
|
||||
scannerManager = newManager(
|
||||
&managerOptions{
|
||||
registrations: registrations,
|
||||
|
||||
createErrorFn: func(reg *scanner.Registration) error {
|
||||
if reg.Name == "Clair" {
|
||||
return errors.Wrap(types.ErrDupRows, "failed to create reg")
|
||||
}
|
||||
|
||||
return nil
|
||||
mgr.On("List", &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"immutable": true,
|
||||
"ex_url__in": []string{
|
||||
"http://scanner-1",
|
||||
"http://scanner-2",
|
||||
},
|
||||
},
|
||||
)
|
||||
}).Return(registrations, nil)
|
||||
mgr.On("Delete", "uuid-1").Return(nil)
|
||||
mgr.On("Delete", "uuid-2").Return(errors.New("DB error"))
|
||||
|
||||
err := RemoveImmutableScanners([]string{
|
||||
"http://scanner-1",
|
||||
"http://scanner-2",
|
||||
})
|
||||
assert.EqualError(t, err, "deleting scanner: uuid-2: DB error")
|
||||
mgr.AssertExpectations(t)
|
||||
})
|
||||
|
||||
assert.Nil(EnsureScanner(&scanner.Registration{Name: "Clair", URL: "reg1"}))
|
||||
assert.Error(EnsureScanner(&scanner.Registration{Name: "Clair", URL: "reg2"}))
|
||||
assert.Nil(EnsureScanner(&scanner.Registration{Name: "Clair", URL: "reg2"}, true))
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user