diff --git a/Makefile b/Makefile index a8309ab42..e8aa5d41f 100644 --- a/Makefile +++ b/Makefile @@ -77,6 +77,7 @@ REGISTRYPROJECTNAME=goharbor DEVFLAG=true NOTARYFLAG=false CLAIRFLAG=false +TRIVYFLAG=false HTTPPROXY= BUILDBIN=false MIGRATORFLAG=false @@ -104,6 +105,8 @@ MIGRATORVERSION=$(VERSIONTAG) REDISVERSION=$(VERSIONTAG) NOTARYMIGRATEVERSION=v3.5.4 CLAIRADAPTERVERSION=v1.0.1 +TRIVYVERSION=v0.4.3 +TRIVYADAPTERVERSION=v0.2.3 # version of chartmuseum CHARTMUSEUMVERSION=v0.9.0 @@ -117,6 +120,8 @@ REGISTRY_VERSION: $(REGISTRYVERSION) NOTARY_VERSION: $(NOTARYVERSION) CLAIR_VERSION: $(CLAIRVERSION) CLAIR_ADAPTER_VERSION: $(CLAIRADAPTERVERSION) +TRIVY_VERSION: $(TRIVYVERSION) +TRIVY_ADAPTER_VERSION: $(TRIVYADAPTERVERSION) CHARTMUSEUM_VERSION: $(CHARTMUSEUMVERSION) endef @@ -193,6 +198,9 @@ endif ifeq ($(CLAIRFLAG), true) PREPARECMD_PARA+= --with-clair endif +ifeq ($(TRIVYFLAG), true) + PREPARECMD_PARA+= --with-trivy +endif # append chartmuseum parameters if set ifeq ($(CHARTFLAG), true) PREPARECMD_PARA+= --with-chartmuseum @@ -276,6 +284,9 @@ endif ifeq ($(CLAIRFLAG), true) DOCKERSAVE_PARA+= goharbor/clair-photon:$(CLAIRVERSION)-$(VERSIONTAG) goharbor/clair-adapter-photon:$(CLAIRADAPTERVERSION)-$(VERSIONTAG) endif +ifeq ($(TRIVYFLAG), true) + DOCKERSAVE_PARA+= goharbor/trivy-adapter-photon:$(TRIVYADAPTERVERSION)-$(VERSIONTAG) +endif ifeq ($(MIGRATORFLAG), true) DOCKERSAVE_PARA+= goharbor/harbor-migrator:$(MIGRATORVERSION) endif @@ -348,14 +359,16 @@ prepare: update_prepare_version build: make -f $(MAKEFILEPATH_PHOTON)/Makefile $(BUILDTARGET) -e DEVFLAG=$(DEVFLAG) -e GOBUILDIMAGE=$(GOBUILDIMAGE) \ - -e REGISTRYVERSION=$(REGISTRYVERSION) -e REGISTRY_SRC_TAG=$(REGISTRY_SRC_TAG) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \ + -e REGISTRYVERSION=$(REGISTRYVERSION) -e REGISTRY_SRC_TAG=$(REGISTRY_SRC_TAG) -e NGINXVERSION=$(NGINXVERSION) \ + -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \ + -e TRIVYVERSION=$(TRIVYVERSION) -e TRIVYADAPTERVERSION=$(TRIVYADAPTERVERSION) \ -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRADAPTERVERSION=$(CLAIRADAPTERVERSION) -e VERSIONTAG=$(VERSIONTAG) \ -e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \ -e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER) \ -e NPM_REGISTRY=$(NPM_REGISTRY) -e BASEIMAGETAG=$(BASEIMAGETAG) build_base_docker: - @for name in chartserver clair clair-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \ + @for name in chartserver clair clair-adapter trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \ echo $$name ; \ $(DOCKERBUILD) --pull -f $(MAKEFILEPATH_PHOTON)/$$name/Dockerfile.base -t goharbor/harbor-$$name-base:$(BASEIMAGETAG) . ; \ $(PUSHSCRIPTPATH)/$(PUSHSCRIPTNAME) goharbor/harbor-$$name-base:$(BASEIMAGETAG) $(REGISTRYUSER) $(REGISTRYPASSWORD) ; \ diff --git a/make/harbor.yml.tmpl b/make/harbor.yml.tmpl index b5df0bac9..6ce719f3a 100644 --- a/make/harbor.yml.tmpl +++ b/make/harbor.yml.tmpl @@ -141,6 +141,7 @@ _version: 1.10.0 # jobservice_db_index: 2 # chartmuseum_db_index: 3 # clair_db_index: 4 +# trivy_db_index: 5 # Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert. # uaa: diff --git a/make/install.sh b/make/install.sh index c06a1220d..1824df256 100755 --- a/make/install.sh +++ b/make/install.sh @@ -10,6 +10,7 @@ set +o noglob usage=$'Please set hostname and other necessary attributes in harbor.yml first. DO NOT use localhost or 127.0.0.1 for hostname, because Harbor needs to be accessed by external clients. Please set --with-notary if needs enable Notary in Harbor, and set ui_url_protocol/ssl_cert/ssl_cert_key in harbor.yml bacause notary must run under https. Please set --with-clair if needs enable Clair in Harbor +Please set --with-tivy if needs enable Trivy in Harbor Please set --with-chartmuseum if needs enable Chartmuseum in Harbor' item=0 @@ -17,6 +18,8 @@ item=0 with_notary=$false # clair is not enabled by default with_clair=$false +# trivy is not enabled by default +with_trivy=$false # chartmuseum is not enabled by default with_chartmuseum=$false @@ -29,6 +32,8 @@ while [ $# -gt 0 ]; do with_notary=true;; --with-clair) with_clair=true;; + --with-trivy) + with_trivy=true;; --with-chartmuseum) with_chartmuseum=true;; *) @@ -70,6 +75,10 @@ if [ $with_clair ] then prepare_para="${prepare_para} --with-clair" fi +if [ $with_trivy ] +then + prepare_para="${prepare_para} --with-trivy" +fi if [ $with_chartmuseum ] then prepare_para="${prepare_para} --with-chartmuseum" diff --git a/make/photon/Makefile b/make/photon/Makefile index 2c23c37b1..ada398107 100644 --- a/make/photon/Makefile +++ b/make/photon/Makefile @@ -60,11 +60,18 @@ DOCKERIMAGENAME_POSTGRESQL=goharbor/postgresql-photon DOCKERFILEPATH_CLAIR=$(DOCKERFILEPATH)/clair DOCKERFILENAME_CLAIR=Dockerfile DOCKERIMAGENAME_CLAIR=goharbor/clair-photon +CLAIR_ADAPTER_DOWNLOAD_URL=https://github.com/goharbor/harbor-scanner-clair/releases/download/$(CLAIRADAPTERVERSION)/harbor-scanner-clair_$(CLAIRADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz DOCKERFILEPATH_CLAIR_ADAPTER=$(DOCKERFILEPATH)/clair-adapter DOCKERFILENAME_CLAIR_ADAPTER=Dockerfile DOCKERIMAGENAME_CLAIR_ADAPTER=goharbor/clair-adapter-photon +DOCKERFILEPATH_TRIVY_ADAPTER=$(DOCKERFILEPATH)/trivy-adapter +DOCKERFILENAME_TRIVY_ADAPTER=Dockerfile +DOCKERIMAGENAME_TRIVY_ADAPTER=goharbor/trivy-adapter-photon +TRIVY_DOWNLOAD_URL=https://github.com/aquasecurity/trivy/releases/download/$(TRIVYVERSION)/trivy_$(TRIVYVERSION:v%=%)_Linux-64bit.tar.gz +TRIVY_ADAPTER_DOWNLOAD_URL=https://github.com/aquasecurity/harbor-scanner-trivy/releases/download/$(TRIVYADAPTERVERSION)/harbor-scanner-trivy_$(TRIVYADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz + DOCKERFILEPATH_NGINX=$(DOCKERFILEPATH)/nginx DOCKERFILENAME_NGINX=Dockerfile DOCKERIMAGENAME_NGINX=goharbor/nginx-photon @@ -148,17 +155,40 @@ _build_clair_adapter: @if [ "$(CLAIRFLAG)" = "true" ] ; then \ if [ "$(BUILDBIN)" != "true" ] ; then \ rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary && mkdir -p $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary && \ - $(call _extract_archive, https://github.com/goharbor/harbor-scanner-clair/releases/download/$(CLAIRADAPTERVERSION)/harbor-scanner-clair_$(CLAIRADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz, $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/) && \ + $(call _extract_archive, $(CLAIR_ADAPTER_DOWNLOAD_URL), $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/) && \ mv $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/scanner-clair $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/harbor-scanner-clair; \ else \ - cd $(DOCKERFILEPATH_CLAIR_ADAPTER) && $(DOCKERFILEPATH_CLAIR_ADAPTER)/builder $(CLAIRADAPTERVERSION) && cd - ; \ + cd $(DOCKERFILEPATH_CLAIR_ADAPTER) && $(DOCKERFILEPATH_CLAIR_ADAPTER)/builder.sh $(CLAIRADAPTERVERSION) && cd - ; \ fi ; \ - echo "building clair adapter container for photon..." ; \ - $(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) -f $(DOCKERFILEPATH_CLAIR_ADAPTER)/$(DOCKERFILENAME_CLAIR_ADAPTER) -t $(DOCKERIMAGENAME_CLAIR_ADAPTER):$(CLAIRADAPTERVERSION)-$(VERSIONTAG) . ; \ + echo "Building Clair adapter container for photon..." ; \ + $(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) \ + -f $(DOCKERFILEPATH_CLAIR_ADAPTER)/$(DOCKERFILENAME_CLAIR_ADAPTER) \ + -t $(DOCKERIMAGENAME_CLAIR_ADAPTER):$(CLAIRADAPTERVERSION)-$(VERSIONTAG) . ; \ rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary; \ echo "Done." ; \ fi +_build_trivy_adapter: + @if [ "$(TRIVYFLAG)" = "true" ] ; then \ + rm -rf $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary && mkdir -p $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary ; \ + echo "Downloading Trivy scanner $(TRIVYVERSION)..." ; \ + $(call _extract_archive, $(TRIVY_DOWNLOAD_URL), $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary/) ; \ + if [ "$(BUILDBIN)" != "true" ] ; then \ + echo "Downloading Trivy adapter $(TRIVYADAPTERVERSION)..." ; \ + $(call _extract_archive, $(TRIVY_ADAPTER_DOWNLOAD_URL), $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary/) ; \ + else \ + echo "Building Trivy adapter $(TRIVYADAPTERVERSION) from sources..." ; \ + cd $(DOCKERFILEPATH_TRIVY_ADAPTER) && $(DOCKERFILEPATH_TRIVY_ADAPTER)/builder.sh $(TRIVYADAPTERVERSION) && cd - ; \ + fi ; \ + echo "Building Trivy adapter container for photon..." ; \ + $(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) \ + --build-arg trivy_version=$(TRIVYVERSION) \ + -f $(DOCKERFILEPATH_TRIVY_ADAPTER)/$(DOCKERFILENAME_TRIVY_ADAPTER) \ + -t $(DOCKERIMAGENAME_TRIVY_ADAPTER):$(TRIVYADAPTERVERSION)-$(VERSIONTAG) . ; \ + rm -rf $(DOCKERFILEPATH_TRIVY_ADAPTER)/binary; \ + echo "Done." ; \ + fi + _build_chart_server: @if [ "$(CHARTFLAG)" = "true" ] ; then \ if [ "$(BUILDBIN)" != "true" ] ; then \ @@ -231,7 +261,7 @@ define _get_binary $(WGET) --timeout 30 --no-check-certificate $1 -O $2 || exit 1 endef -build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_clair_adapter _build_redis _build_migrator _build_chart_server +build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_clair_adapter _build_trivy_adapter _build_redis _build_migrator _build_chart_server cleanimage: @echo "cleaning image for photon..." diff --git a/make/photon/clair-adapter/builder b/make/photon/clair-adapter/builder.sh similarity index 65% rename from make/photon/clair-adapter/builder rename to make/photon/clair-adapter/builder.sh index 4ab97c949..ec5e029fa 100755 --- a/make/photon/clair-adapter/builder +++ b/make/photon/clair-adapter/builder.sh @@ -15,25 +15,25 @@ set -e mkdir -p binary rm -rf binary/harbor-scanner-clair || true -cd `dirname $0` +cd $(dirname $0) cur=$PWD -# the temp folder to store distribution source code... -TEMP=`mktemp -d ${TMPDIR-/tmp}/clair-adapter.XXXXXX` +# The temporary directory to clone Clair adapter source code +TEMP=$(mktemp -d ${TMPDIR-/tmp}/clair-adapter.XXXXXX) git clone https://github.com/goharbor/harbor-scanner-clair.git $TEMP cd $TEMP; git checkout $VERSION; cd - -echo 'build the clair adapter binary bases on the golang:1.13.4' +echo "Building Clair adapter binary based on golang:1.13.4..." cp Dockerfile.binary $TEMP docker build -f $TEMP/Dockerfile.binary -t clair-adapter-golang $TEMP -echo 'copy the clair adapter binary to local...' +echo "Copying Clair adapter binary from the container to the local directory..." ID=$(docker create clair-adapter-golang) docker cp $ID:/go/src/github.com/goharbor/harbor-scanner-clair/harbor-scanner-clair binary docker rm -f $ID docker rmi -f clair-adapter-golang -echo "Build clair adapter binary success, then to build photon image..." +echo "Building Clair adapter binary finished successfully" cd $cur rm -rf $TEMP diff --git a/make/photon/prepare/main.py b/make/photon/prepare/main.py index 2439e6f33..9028a1a13 100644 --- a/make/photon/prepare/main.py +++ b/make/photon/prepare/main.py @@ -15,6 +15,7 @@ from utils.notary import prepare_notary from utils.log import prepare_log_configs from utils.clair import prepare_clair from utils.clair_adapter import prepare_clair_adapter +from utils.trivy_adapter import prepare_trivy_adapter from utils.chart import prepare_chartmuseum from utils.docker_compose import prepare_docker_compose from utils.nginx import prepare_nginx, nginx_confd_dir @@ -27,11 +28,12 @@ old_private_key_pem_path, old_crt_path) @click.option('--conf', default=input_config_path, help="the path of Harbor configuration file") @click.option('--with-notary', is_flag=True, help="the Harbor instance is to be deployed with notary") @click.option('--with-clair', is_flag=True, help="the Harbor instance is to be deployed with clair") +@click.option('--with-trivy', is_flag=True, help="the Harbor instance is to be deployed with Trivy") @click.option('--with-chartmuseum', is_flag=True, help="the Harbor instance is to be deployed with chart repository supporting") -def main(conf, with_notary, with_clair, with_chartmuseum): +def main(conf, with_notary, with_clair, with_trivy, with_chartmuseum): delfile(config_dir) - config_dict = parse_yaml_config(conf, with_notary=with_notary, with_clair=with_clair, with_chartmuseum=with_chartmuseum) + config_dict = parse_yaml_config(conf, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum) try: validate(config_dict, notary_mode=with_notary) except Exception as e: @@ -41,7 +43,7 @@ def main(conf, with_notary, with_clair, with_chartmuseum): prepare_log_configs(config_dict) prepare_nginx(config_dict) - prepare_core(config_dict, with_notary=with_notary, with_clair=with_clair, with_chartmuseum=with_chartmuseum) + prepare_core(config_dict, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum) prepare_registry(config_dict) prepare_registry_ctl(config_dict) prepare_db(config_dict) @@ -63,10 +65,13 @@ def main(conf, with_notary, with_clair, with_chartmuseum): prepare_clair(config_dict) prepare_clair_adapter(config_dict) + if with_trivy: + prepare_trivy_adapter(config_dict) + if with_chartmuseum: prepare_chartmuseum(config_dict) - prepare_docker_compose(config_dict, with_clair, with_notary, with_chartmuseum) + prepare_docker_compose(config_dict, with_clair, with_trivy, with_notary, with_chartmuseum) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/make/photon/prepare/templates/core/env.jinja b/make/photon/prepare/templates/core/env.jinja index c965a9dbb..0c01a9c79 100644 --- a/make/photon/prepare/templates/core/env.jinja +++ b/make/photon/prepare/templates/core/env.jinja @@ -26,6 +26,7 @@ CORE_SECRET={{core_secret}} JOBSERVICE_SECRET={{jobservice_secret}} WITH_NOTARY={{with_notary}} WITH_CLAIR={{with_clair}} +WITH_TRIVY={{with_trivy}} CLAIR_DB_PASSWORD={{clair_db_password}} CLAIR_DB_HOST={{clair_db_host}} CLAIR_DB_PORT={{clair_db_port}} @@ -37,6 +38,7 @@ CORE_LOCAL_URL={{core_local_url}} JOBSERVICE_URL={{jobservice_url}} CLAIR_URL={{clair_url}} CLAIR_ADAPTER_URL={{clair_adapter_url}} +TRIVY_ADAPTER_URL={{trivy_adapter_url}} NOTARY_URL={{notary_url}} REGISTRY_STORAGE_PROVIDER_NAME={{storage_provider_name}} READ_ONLY=false diff --git a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja index 1aabb6f8b..167aa4f6c 100644 --- a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja +++ b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja @@ -437,6 +437,35 @@ services: env_file: ./common/config/clair-adapter/env {% endif %} +{% if with_trivy %} + trivy-adapter: + container_name: trivy-adapter + image: goharbor/trivy-adapter-photon:{{trivy_adapter_version}} + restart: always + cap_drop: + - ALL + dns_search: . + networks: + - harbor +{% if external_redis == False %} + depends_on: + - redis +{% endif %} + volumes: + - type: bind + source: {{data_volume}}/trivy-adapter/trivy + target: /home/scanner/.cache/trivy + - type: bind + source: {{data_volume}}/trivy-adapter/reports + target: /home/scanner/.cache/reports + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + tag: "trivy-adapter" + env_file: + ./common/config/trivy-adapter/env +{% endif %} {% if with_chartmuseum %} chartmuseum: container_name: chartmuseum @@ -491,4 +520,4 @@ networks: {% if with_chartmuseum %} harbor-chartmuseum: external: false -{% endif %} \ No newline at end of file +{% endif %} diff --git a/make/photon/prepare/templates/trivy-adapter/env.jinja b/make/photon/prepare/templates/trivy-adapter/env.jinja new file mode 100644 index 000000000..9bdb20e93 --- /dev/null +++ b/make/photon/prepare/templates/trivy-adapter/env.jinja @@ -0,0 +1,10 @@ +SCANNER_LOG_LEVEL={{log_level}} +SCANNER_STORE_REDIS_URL={{redis_url_trivy}} +SCANNER_STORE_REDIS_NAMESPACE=harbor.scanner.trivy:store +SCANNER_JOB_QUEUE_REDIS_URL={{redis_url_trivy}} +SCANNER_JOB_QUEUE_REDIS_NAMESPACE=harbor.scanner.trivy:job-queue +SCANNER_TRIVY_CACHE_DIR=/home/scanner/.cache/trivy +SCANNER_TRIVY_REPORTS_DIR=/home/scanner/.cache/reports +SCANNER_TRIVY_VULN_TYPE=os,library +SCANNER_TRIVY_SEVERITY=UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL +SCANNER_TRIVY_IGNORE_UNFIXED=false diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index df24f8122..ee785a1bc 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -94,7 +94,7 @@ def parse_versions(): return versions -def parse_yaml_config(config_file_path, with_notary, with_clair, with_chartmuseum): +def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, with_chartmuseum): ''' :param configs: config_parser object :returns: dict of configs @@ -113,6 +113,7 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_chartmuseu 'jobservice_url': 'http://jobservice:8080', 'clair_url': 'http://clair:6060', 'clair_adapter_url': 'http://clair-adapter:8080', + 'trivy_adapter_url': 'http://trivy-adapter:8080', 'notary_url': 'http://notary-server:4443', 'chart_repository_url': 'http://chartmuseum:9999' } @@ -317,7 +318,7 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_chartmuseu config_dict['external_database'] = False # update redis configs - config_dict.update(get_redis_configs(configs.get("external_redis", None), with_clair)) + config_dict.update(get_redis_configs(configs.get("external_redis", None), with_clair, with_trivy)) # auto generated secret string for core config_dict['core_secret'] = generate_random_string(16) @@ -351,7 +352,7 @@ def get_redis_url(db, redis=None): return "redis://{host}:{port}/{db}".format(**kwargs) -def get_redis_configs(external_redis=None, with_clair=True): +def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True): """Returns configs for redis >>> get_redis_configs()['external_redis'] @@ -362,6 +363,8 @@ def get_redis_configs(external_redis=None, with_clair=True): 'redis://redis:6379/2' >>> get_redis_configs()['redis_url_clair'] 'redis://redis:6379/4' + >>> get_redis_configs()['redis_url_trivy'] + 'redis://redis:6379/5' >>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['external_redis'] True @@ -371,9 +374,13 @@ def get_redis_configs(external_redis=None, with_clair=True): 'redis://anonymous:pass@localhost:6379/2' >>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_clair'] 'redis://anonymous:pass@localhost:6379/4' + >>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_trivy'] + 'redis://anonymous:pass@localhost:6379/5' >>> 'redis_url_clair' not in get_redis_configs(with_clair=False) True + >>> 'redis_url_trivy' not in get_redis_configs(with_trivy=False) + True """ configs = dict(external_redis=bool(external_redis)) @@ -387,6 +394,7 @@ def get_redis_configs(external_redis=None, with_clair=True): 'jobservice_db_index': 2, 'chartmuseum_db_index': 3, 'clair_db_index': 4, + 'trivy_db_index': 5, } # overwriting existing keys by external_redis @@ -406,4 +414,8 @@ def get_redis_configs(external_redis=None, with_clair=True): configs['redis_db_index_clair'] = redis['clair_db_index'] configs['redis_url_clair'] = get_redis_url(configs['redis_db_index_clair'], redis) + if with_trivy: + configs['redis_db_index_trivy'] = redis['trivy_db_index'] + configs['redis_url_trivy'] = get_redis_url(configs['redis_db_index_trivy'], redis) + return configs diff --git a/make/photon/prepare/utils/core.py b/make/photon/prepare/utils/core.py index a1dd7aa46..4d669a954 100644 --- a/make/photon/prepare/utils/core.py +++ b/make/photon/prepare/utils/core.py @@ -13,7 +13,7 @@ core_conf = os.path.join(config_dir, "core", "app.conf") ca_download_dir = os.path.join(data_dir, 'ca_download') -def prepare_core(config_dict, with_notary, with_clair, with_chartmuseum): +def prepare_core(config_dict, with_notary, with_clair, with_trivy, with_chartmuseum): prepare_dir(ca_download_dir, uid=DEFAULT_UID, gid=DEFAULT_GID) prepare_dir(core_config_dir) # Render Core @@ -30,6 +30,7 @@ def prepare_core(config_dict, with_notary, with_clair, with_chartmuseum): chart_cache_driver=chart_cache_driver, with_notary=with_notary, with_clair=with_clair, + with_trivy=with_trivy, with_chartmuseum=with_chartmuseum, **config_dict) @@ -41,7 +42,6 @@ def prepare_core(config_dict, with_notary, with_clair, with_chartmuseum): xsrf_key=generate_random_string(40)) - def copy_core_config(core_templates_path, core_config_path): shutil.copyfile(core_templates_path, core_config_path) print("Generated configuration file: %s" % core_config_path) diff --git a/make/photon/prepare/utils/docker_compose.py b/make/photon/prepare/utils/docker_compose.py index 7716b7e39..6667e82fb 100644 --- a/make/photon/prepare/utils/docker_compose.py +++ b/make/photon/prepare/utils/docker_compose.py @@ -8,13 +8,14 @@ docker_compose_template_path = os.path.join(templates_dir, 'docker_compose', 'do docker_compose_yml_path = '/compose_location/docker-compose.yml' # render docker-compose -def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum): +def prepare_docker_compose(configs, with_clair, with_trivy, with_notary, with_chartmuseum): versions = parse_versions() VERSION_TAG = versions.get('VERSION_TAG') or 'dev' REGISTRY_VERSION = versions.get('REGISTRY_VERSION') or 'v2.7.1-patch-2819-2553' NOTARY_VERSION = versions.get('NOTARY_VERSION') or 'v0.6.1' CLAIR_VERSION = versions.get('CLAIR_VERSION') or 'v2.0.9' CLAIR_ADAPTER_VERSION = versions.get('CLAIR_ADAPTER_VERSION') or 'v1.0.0' + TRIVY_ADAPTER_VERSION = versions.get('TRIVY_ADAPTER_VERSION') or 'v0.2.3' CHARTMUSEUM_VERSION = versions.get('CHARTMUSEUM_VERSION') or 'v0.9.0' rendering_variables = { @@ -24,6 +25,7 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum): 'notary_version': '{}-{}'.format(NOTARY_VERSION, VERSION_TAG), 'clair_version': '{}-{}'.format(CLAIR_VERSION, VERSION_TAG), 'clair_adapter_version': '{}-{}'.format(CLAIR_ADAPTER_VERSION, VERSION_TAG), + 'trivy_adapter_version': '{}-{}'.format(TRIVY_ADAPTER_VERSION, VERSION_TAG), 'chartmuseum_version': '{}-{}'.format(CHARTMUSEUM_VERSION, VERSION_TAG), 'data_volume': configs['data_volume'], 'log_location': configs['log_location'], @@ -34,6 +36,7 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum): 'external_database': configs['external_database'], 'with_notary': with_notary, 'with_clair': with_clair, + 'with_trivy': with_trivy, 'with_chartmuseum': with_chartmuseum } diff --git a/make/photon/prepare/utils/trivy_adapter.py b/make/photon/prepare/utils/trivy_adapter.py new file mode 100644 index 000000000..c314c1a67 --- /dev/null +++ b/make/photon/prepare/utils/trivy_adapter.py @@ -0,0 +1,21 @@ +import os + +from g import templates_dir, config_dir, data_dir, DEFAULT_UID, DEFAULT_GID +from .jinja import render_jinja +from .misc import prepare_dir + +trivy_adapter_template_dir = os.path.join(templates_dir, "trivy-adapter") + + +def prepare_trivy_adapter(config_dict): + trivy_adapter_config_dir = prepare_dir(config_dir, "trivy-adapter") + prepare_dir(data_dir, "trivy-adapter", "trivy", uid=DEFAULT_UID, gid=DEFAULT_GID) + prepare_dir(data_dir, "trivy-adapter", "reports", uid=DEFAULT_UID, gid=DEFAULT_GID) + + trivy_adapter_env_path = os.path.join(trivy_adapter_config_dir, "env") + trivy_adapter_env_template = os.path.join(trivy_adapter_template_dir, "env.jinja") + + render_jinja( + trivy_adapter_env_template, + trivy_adapter_env_path, + **config_dict) diff --git a/make/photon/trivy-adapter/Dockerfile b/make/photon/trivy-adapter/Dockerfile new file mode 100644 index 000000000..fc21b698b --- /dev/null +++ b/make/photon/trivy-adapter/Dockerfile @@ -0,0 +1,17 @@ +ARG harbor_base_image_version +FROM goharbor/harbor-trivy-adapter-base:${harbor_base_image_version} + +ARG trivy_version + +COPY ./make/photon/trivy-adapter/binary/trivy /usr/local/bin/trivy +COPY ./make/photon/trivy-adapter/binary/scanner-trivy /home/scanner/bin/scanner-trivy + +EXPOSE 8080 + +HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS 127.0.0.1:8080/probe/healthy || exit 1 + +ENV TRIVY_VERSION=${trivy_version} + +USER scanner + +ENTRYPOINT ["/home/scanner/bin/scanner-trivy"] diff --git a/make/photon/trivy-adapter/Dockerfile.base b/make/photon/trivy-adapter/Dockerfile.base new file mode 100644 index 000000000..3b38817dc --- /dev/null +++ b/make/photon/trivy-adapter/Dockerfile.base @@ -0,0 +1,6 @@ +FROM photon:2.0 + +RUN tdnf install -y sudo rpm >> /dev/null \ + && tdnf clean all \ + && groupadd -r -g 10000 scanner \ + && useradd --no-log-init -m -r -g 10000 -u 10000 scanner diff --git a/make/photon/trivy-adapter/Dockerfile.binary b/make/photon/trivy-adapter/Dockerfile.binary new file mode 100644 index 000000000..e8c4e1772 --- /dev/null +++ b/make/photon/trivy-adapter/Dockerfile.binary @@ -0,0 +1,7 @@ +FROM golang:1.13.4 + +ADD . /go/src/github.com/aquasecurity/harbor-scanner-trivy/ +WORKDIR /go/src/github.com/aquasecurity/harbor-scanner-trivy/ + +RUN export GOOS=linux GO111MODULE=on CGO_ENABLED=0 && \ + go build -o scanner-trivy cmd/scanner-trivy/main.go diff --git a/make/photon/trivy-adapter/builder.sh b/make/photon/trivy-adapter/builder.sh new file mode 100755 index 000000000..516c1d164 --- /dev/null +++ b/make/photon/trivy-adapter/builder.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set +e + +if [ -z $1 ]; then + error "Please set the 'version' variable" + exit 1 +fi + +VERSION="$1" + +set -e + +cd $(dirname $0) +cur=$PWD + +# The temporary directory to clone Trivy adapter source code +TEMP=$(mktemp -d ${TMPDIR-/tmp}/trivy-adapter.XXXXXX) +git clone https://github.com/aquasecurity/harbor-scanner-trivy.git $TEMP +cd $TEMP; git checkout $VERSION; cd - + +echo "Building Trivy adapter binary based on golang:1.13.4..." +cp Dockerfile.binary $TEMP +docker build -f $TEMP/Dockerfile.binary -t trivy-adapter-golang $TEMP + +echo "Copying Trivy adapter binary from the container to the local directory..." +ID=$(docker create trivy-adapter-golang) +docker cp $ID:/go/src/github.com/aquasecurity/harbor-scanner-trivy/scanner-trivy binary + +docker rm -f $ID +docker rmi -f trivy-adapter-golang + +echo "Building Trivy adapter binary finished successfully" +cd $cur +rm -rf $TEMP diff --git a/src/common/config/metadata/metadatalist.go b/src/common/config/metadata/metadatalist.go index 07bbd0d0a..17216060d 100644 --- a/src/common/config/metadata/metadatalist.go +++ b/src/common/config/metadata/metadatalist.go @@ -73,6 +73,7 @@ var ( {Name: common.ClairDBUsername, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_DB_USERNAME", DefaultValue: "postgres", ItemType: &StringType{}, Editable: false}, {Name: common.ClairURL, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_URL", DefaultValue: "http://clair:6060", ItemType: &StringType{}, Editable: false}, {Name: common.ClairAdapterURL, Scope: SystemScope, Group: ClairGroup, EnvKey: "CLAIR_ADAPTER_URL", DefaultValue: "http://clair-adapter:8080", ItemType: &StringType{}, Editable: false}, + {Name: common.TrivyAdapterURL, Scope: SystemScope, Group: ClairGroup, EnvKey: "TRIVY_ADAPTER_URL", DefaultValue: "http://trivy-adapter:8080", ItemType: &StringType{}, Editable: false}, {Name: common.CoreURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "CORE_URL", DefaultValue: "http://core:8080", ItemType: &StringType{}, Editable: false}, {Name: common.CoreLocalURL, Scope: SystemScope, Group: BasicGroup, EnvKey: "CORE_LOCAL_URL", DefaultValue: "http://127.0.0.1:8080", ItemType: &StringType{}, Editable: false}, @@ -150,6 +151,7 @@ var ( {Name: common.WithChartMuseum, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_CHARTMUSEUM", DefaultValue: "false", ItemType: &BoolType{}, Editable: true}, {Name: common.WithClair, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_CLAIR", DefaultValue: "false", ItemType: &BoolType{}, Editable: true}, + {Name: common.WithTrivy, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_TRIVY", DefaultValue: "false", ItemType: &BoolType{}, Editable: true}, {Name: common.WithNotary, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_NOTARY", DefaultValue: "false", ItemType: &BoolType{}, Editable: true}, // the unit of expiration is minute, 43200 minutes = 30 days {Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "43200", ItemType: &IntType{}, Editable: true}, diff --git a/src/common/const.go b/src/common/const.go index 1dc3937df..09be9fe31 100755 --- a/src/common/const.go +++ b/src/common/const.go @@ -89,6 +89,7 @@ const ( AdminInitialPassword = "admin_initial_password" WithNotary = "with_notary" WithClair = "with_clair" + WithTrivy = "with_trivy" ScanAllPolicy = "scan_all_policy" ClairDBPassword = "clair_db_password" ClairDBHost = "clair_db_host" @@ -123,6 +124,7 @@ const ( ReadOnly = "read_only" ClairURL = "clair_url" ClairAdapterURL = "clair_adapter_url" + TrivyAdapterURL = "trivy_adapter_url" NotaryURL = "notary_url" DefaultCoreEndpoint = "http://core:8080" DefaultNotaryEndpoint = "http://notary-server:4443" diff --git a/src/core/config/config.go b/src/core/config/config.go index e40d7a7c8..0c18486b0 100755 --- a/src/core/config/config.go +++ b/src/core/config/config.go @@ -348,6 +348,16 @@ func ClairAdapterEndpoint() string { return cfgMgr.Get(common.ClairAdapterURL).GetString() } +// WithTrivy returns a bool value to indicate if Harbor's deployed with Trivy. +func WithTrivy() bool { + return cfgMgr.Get(common.WithTrivy).GetBool() +} + +// TrivyAdapterURL returns the endpoint URL of a Trivy adapter instance, by default it's the one deployed within Harbor. +func TrivyAdapterURL() string { + return cfgMgr.Get(common.TrivyAdapterURL).GetString() +} + // UAASettings returns the UAASettings to access UAA service. func UAASettings() (*models.UAASettings, error) { err := cfgMgr.Load() diff --git a/src/core/main.go b/src/core/main.go index cd389549e..1cda4d598 100755 --- a/src/core/main.go +++ b/src/core/main.go @@ -208,31 +208,7 @@ func main() { log.Fatalf("Failed to initialize API handlers with error: %s", err.Error()) } - if config.WithClair() { - clairDB, err := config.ClairDB() - if err != nil { - log.Fatalf("failed to load clair database information: %v", err) - } - if err := dao.InitClairDB(clairDB); err != nil { - log.Fatalf("failed to initialize clair database: %v", err) - } - - reg := &scanner.Registration{ - Name: "Clair", - Description: "The clair scanner adapter", - URL: config.ClairAdapterEndpoint(), - UseInternalAddr: true, - Immutable: true, - } - - if err := scan.EnsureScanner(reg, true); err != nil { - log.Fatalf("failed to initialize clair scanner: %v", err) - } - } else { - if err := scan.RemoveImmutableScanners(); err != nil { - log.Warningf("failed to remove immutable scanners: %v", err) - } - } + registerScanners() closing := make(chan struct{}) done := make(chan struct{}) @@ -291,3 +267,70 @@ func main() { beego.RunWithMiddleWares("", middlewares.MiddleWares()...) } + +func registerScanners() { + wantedScanners := make([]scanner.Registration, 0) + uninstallURLs := make([]string, 0) + + if config.WithTrivy() { + log.Info("Registering Trivy scanner") + wantedScanners = append(wantedScanners, scanner.Registration{ + Name: "Trivy", + Description: "The Trivy scanner adapter", + URL: config.TrivyAdapterURL(), + UseInternalAddr: true, + Immutable: true, + }) + } else { + log.Info("Removing Trivy scanner") + uninstallURLs = append(uninstallURLs, config.TrivyAdapterURL()) + } + + if config.WithClair() { + clairDB, err := config.ClairDB() + if err != nil { + log.Fatalf("failed to load clair database information: %v", err) + } + if err := dao.InitClairDB(clairDB); err != nil { + log.Fatalf("failed to initialize clair database: %v", err) + } + + log.Info("Registering Clair scanner") + wantedScanners = append(wantedScanners, scanner.Registration{ + Name: "Clair", + Description: "The Clair scanner adapter", + URL: config.ClairAdapterEndpoint(), + UseInternalAddr: true, + Immutable: true, + }) + } else { + log.Info("Removing Clair scanner") + uninstallURLs = append(uninstallURLs, config.ClairAdapterEndpoint()) + } + + if err := scan.EnsureScanners(wantedScanners); err != nil { + log.Fatalf("failed to register scanners: %v", err) + } + + if defaultScannerURL := getDefaultScannerURL(); defaultScannerURL != "" { + log.Infof("Setting %s as default scanner", defaultScannerURL) + if err := scan.EnsureDefaultScanner(defaultScannerURL); err != nil { + log.Fatalf("failed to set default scanner: %v", err) + } + } + + if err := scan.RemoveImmutableScanners(uninstallURLs); err != nil { + log.Warningf("failed to remove scanners: %v", err) + } + +} + +func getDefaultScannerURL() string { + if config.WithTrivy() { + return config.TrivyAdapterURL() + } + if config.WithClair() { + return config.ClairAdapterEndpoint() + } + return "" +} diff --git a/src/pkg/scan/init.go b/src/pkg/scan/init.go index a0288ac88..347740f60 100644 --- a/src/pkg/scan/init.go +++ b/src/pkg/scan/init.go @@ -15,8 +15,6 @@ package scan import ( - "fmt" - "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/pkg/q" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" @@ -30,36 +28,73 @@ var ( scannerManager = sc.New() ) -// EnsureScanner ensure the scanner which specially endpoint exists in the system -func EnsureScanner(registration *scanner.Registration, resolveConflicts ...bool) error { - q := &q.Query{ - Keywords: map[string]interface{}{"url": registration.URL}, +// EnsureScanners ensures that the scanners with the specified endpoints URLs exist in the system. +func EnsureScanners(wantedScanners []scanner.Registration) (err error) { + if len(wantedScanners) == 0 { + return + } + endpointURLs := make([]string, len(wantedScanners)) + for i, ws := range wantedScanners { + endpointURLs[i] = ws.URL } - // Check if the registration with the url already existing. - registrations, err := scannerManager.List(q) + list, err := scannerManager.List(&q.Query{ + Keywords: map[string]interface{}{ + "ex_url__in": endpointURLs, + }, + }) if err != nil { - return err + return errors.Errorf("listing scanners: %v", err) + } + existingScanners := make(map[string]*scanner.Registration) + for _, li := range list { + existingScanners[li.URL] = li } - if len(registrations) > 0 { - return nil + for _, ws := range wantedScanners { + if _, exists := existingScanners[ws.URL]; exists { + log.Infof("Scanner registration already exists: %s", ws.URL) + continue + } + err = createRegistration(&ws, true) + if err != nil { + return errors.Errorf("creating registration: %s: %v", ws.URL, err) + } + log.Infof("Successfully registered %s scanner at %s", ws.Name, ws.URL) } - var resolveConflict bool - if len(resolveConflicts) > 0 { - resolveConflict = resolveConflicts[0] - } + return +} - var defaultReg *scanner.Registration - defaultReg, err = scannerManager.GetDefault() +// EnsureDefaultScanner ensures that the scanner with the specified URL is set as default in the system. +func EnsureDefaultScanner(scannerURL string) (err error) { + defaultScanner, err := scannerManager.GetDefault() if err != nil { - return fmt.Errorf("failed to get the default scanner, error: %v", err) + err = errors.Errorf("getting default scanner: %v", err) + return } + if defaultScanner != nil && defaultScanner.URL == scannerURL { + log.Infof("The default scanner is already set: %s", defaultScanner.URL) + return + } + scanners, err := scannerManager.List(&q.Query{ + Keywords: map[string]interface{}{"url": scannerURL}, + }) + if err != nil { + err = errors.Errorf("listing scanners: %v", err) + return + } + if len(scanners) != 1 { + return errors.Errorf("expected only one scanner with URL %v but got %d", scannerURL, len(scanners)) + } + err = scannerManager.SetAsDefault(scanners[0].UUID) + if err != nil { + err = errors.Errorf("setting %s as default scanner: %v", scannerURL, err) + } + return +} - // Set the registration to be default one when no default registration exist in the system - registration.IsDefault = defaultReg == nil - +func createRegistration(registration *scanner.Registration, resolveConflict bool) (err error) { for { _, err = scannerManager.Create(registration) if err != nil { @@ -78,28 +113,30 @@ func EnsureScanner(registration *scanner.Registration, resolveConflicts ...bool) break } - - if err == nil { - log.Infof("initialized scanner named %s", registration.Name) - } - - return err + return } -// RemoveImmutableScanners remove all immutable scanners in the system -func RemoveImmutableScanners() error { - q := &q.Query{ - Keywords: map[string]interface{}{"immutable": true}, +// RemoveImmutableScanners removes immutable scanner Registrations with the specified endpoint URLs. +func RemoveImmutableScanners(urls []string) error { + if len(urls) == 0 { + return nil + } + query := &q.Query{ + Keywords: map[string]interface{}{ + "immutable": true, + "ex_url__in": urls, + }, } - registrations, err := scannerManager.List(q) + // TODO Instead of executing 1 to N SQL queries we might want to delete multiple rows with scannerManager.DeleteByImmutableAndURLIn(true, []string{}) + registrations, err := scannerManager.List(query) if err != nil { - return err + return errors.Errorf("listing scanners: %v", err) } for _, reg := range registrations { if err := scannerManager.Delete(reg.UUID); err != nil { - return err + return errors.Errorf("deleting scanner: %s: %v", reg.UUID, err) } } diff --git a/src/pkg/scan/init_test.go b/src/pkg/scan/init_test.go index b95bd1369..aabe4af22 100644 --- a/src/pkg/scan/init_test.go +++ b/src/pkg/scan/init_test.go @@ -15,201 +15,260 @@ package scan import ( - "testing" - "github.com/goharbor/harbor/src/pkg/q" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" - sc "github.com/goharbor/harbor/src/pkg/scan/scanner" "github.com/goharbor/harbor/src/pkg/scan/scanner/mocks" - "github.com/goharbor/harbor/src/pkg/types" "github.com/pkg/errors" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" + "testing" ) -type managerOptions struct { - registrations []*scanner.Registration - listError error - getError error - getDefaultError error - createError error - createErrorFn func(*scanner.Registration) error -} +func TestEnsureScanners(t *testing.T) { -func newManager(opts *managerOptions) sc.Manager { - if opts == nil { - opts = &managerOptions{} - } + t.Run("Should do nothing when list of wanted scanners is empty", func(t *testing.T) { + err := EnsureScanners([]scanner.Registration{}) + assert.NoError(t, err) + }) - data := map[string]*scanner.Registration{} - for _, reg := range opts.registrations { - data[reg.URL] = reg - } + t.Run("Should return error when listing scanners fails", func(t *testing.T) { + mgr := &mocks.Manager{} + scannerManager = mgr - mgr := &mocks.Manager{} - - listFn := func(query *q.Query) []*scanner.Registration { - if opts.listError != nil { - return nil - } - - url := query.Keywords["url"] - - var results []*scanner.Registration - for key, reg := range data { - if url == key { - results = append(results, reg) - } - } - - return results - } - - getFn := func(url string) *scanner.Registration { - if opts.getError != nil { - return nil - } - - return data[url] - } - - getDefaultFn := func() *scanner.Registration { - if opts.getDefaultError != nil { - return nil - } - - for _, reg := range data { - if reg.IsDefault { - return reg - } - } - - return nil - } - - createFn := func(reg *scanner.Registration) string { - if opts.createError != nil { - return "" - } - - data[reg.URL] = reg - - return reg.URL - } - - createError := func(reg *scanner.Registration) error { - if opts.createErrorFn != nil { - return opts.createErrorFn(reg) - } - - return opts.createError - } - - mgr.On("List", mock.AnythingOfType("*q.Query")).Return(listFn, opts.listError) - mgr.On("Get", mock.AnythingOfType("string")).Return(getFn, opts.getError) - mgr.On("GetDefault").Return(getDefaultFn, opts.getDefaultError) - mgr.On("Create", mock.AnythingOfType("*scanner.Registration")).Return(createFn, createError) - - return mgr -} - -func TestEnsureScanner(t *testing.T) { - assert := assert.New(t) - - registrations := []*scanner.Registration{ - {URL: "reg1"}, - } - - // registration with the url exist in the system - scannerManager = newManager( - &managerOptions{ - registrations: registrations, - }, - ) - assert.Nil(EnsureScanner(&scanner.Registration{URL: "reg1"})) - - // list registrations got error - scannerManager = newManager( - &managerOptions{ - listError: errors.New("list registrations internal error"), - }, - ) - assert.Error(EnsureScanner(&scanner.Registration{URL: "reg1"})) - - // create registration got error - scannerManager = newManager( - &managerOptions{ - createError: errors.New("create registration internal error"), - }, - ) - assert.Error(EnsureScanner(&scanner.Registration{URL: "reg1"})) - - // get default registration got error - scannerManager = newManager( - &managerOptions{ - getDefaultError: errors.New("get default registration internal error"), - }, - ) - assert.Error(EnsureScanner(&scanner.Registration{URL: "reg1"})) - - // create registration when no registrations in the system - scannerManager = newManager(nil) - assert.Nil(EnsureScanner(&scanner.Registration{URL: "reg1"})) - reg1, err := scannerManager.Get("reg1") - assert.Nil(err) - assert.NotNil(reg1) - assert.True(reg1.IsDefault) - - // create registration when there are registrations in the system - scannerManager = newManager( - &managerOptions{ - registrations: registrations, - }, - ) - assert.Nil(EnsureScanner(&scanner.Registration{URL: "reg2"})) - reg2, err := scannerManager.Get("reg2") - assert.Nil(err) - assert.NotNil(reg2) - assert.True(reg2.IsDefault) - - // create registration when there are registrations in the system and the default registration exist - scannerManager = newManager( - &managerOptions{ - registrations: []*scanner.Registration{ - {URL: "reg1", IsDefault: true}, + mgr.On("List", &q.Query{ + Keywords: map[string]interface{}{ + "ex_url__in": []string{"http://scanner:8080"}, }, - }, - ) - assert.Nil(EnsureScanner(&scanner.Registration{URL: "reg3"})) - reg3, err := scannerManager.Get("reg3") - assert.Nil(err) - assert.NotNil(reg3) - assert.False(reg3.IsDefault) -} + }).Return(nil, errors.New("DB error")) -func TestEnsureScannerWithResolveConflict(t *testing.T) { - assert := assert.New(t) + err := EnsureScanners([]scanner.Registration{ + {URL: "http://scanner:8080"}, + }) - registrations := []*scanner.Registration{ - {URL: "reg1"}, - } + assert.EqualError(t, err, "listing scanners: DB error") + mgr.AssertExpectations(t) + }) - // create registration got ErrDupRows when its name is Clair - scannerManager = newManager( - &managerOptions{ - registrations: registrations, + t.Run("Should create only non-existing scanners", func(t *testing.T) { + mgr := &mocks.Manager{} + scannerManager = mgr - createErrorFn: func(reg *scanner.Registration) error { - if reg.Name == "Clair" { - return errors.Wrap(types.ErrDupRows, "failed to create reg") - } - - return nil + mgr.On("List", &q.Query{ + Keywords: map[string]interface{}{ + "ex_url__in": []string{ + "http://trivy:8080", + "http://clair:8080", + }, }, - }, - ) + }).Return([]*scanner.Registration{ + {URL: "http://clair:8080"}, + }, nil) + mgr.On("Create", &scanner.Registration{ + URL: "http://trivy:8080", + }).Return("uuid-trivy", nil) + + err := EnsureScanners([]scanner.Registration{ + {URL: "http://trivy:8080"}, + {URL: "http://clair:8080"}, + }) + + assert.NoError(t, err) + mgr.AssertExpectations(t) + }) + +} + +func TestEnsureDefaultScanner(t *testing.T) { + + t.Run("Should return error when getting default scanner fails", func(t *testing.T) { + mgr := &mocks.Manager{} + scannerManager = mgr + + mgr.On("GetDefault").Return(nil, errors.New("DB error")) + + err := EnsureDefaultScanner("http://trivy:8080") + assert.EqualError(t, err, "getting default scanner: DB error") + mgr.AssertExpectations(t) + }) + + t.Run("Should do nothing when the default scanner is already set", func(t *testing.T) { + mgr := &mocks.Manager{} + scannerManager = mgr + + mgr.On("GetDefault").Return(&scanner.Registration{ + URL: "http://trivy:8080", + }, nil) + + err := EnsureDefaultScanner("http://trivy:8080") + assert.NoError(t, err) + mgr.AssertExpectations(t) + }) + + t.Run("Should return error when listing scanners fails", func(t *testing.T) { + mgr := &mocks.Manager{} + scannerManager = mgr + + mgr.On("GetDefault").Return(nil, nil) + mgr.On("List", &q.Query{ + Keywords: map[string]interface{}{"url": "http://trivy:8080"}, + }).Return(nil, errors.New("DB error")) + + err := EnsureDefaultScanner("http://trivy:8080") + assert.EqualError(t, err, "listing scanners: DB error") + mgr.AssertExpectations(t) + }) + + t.Run("Should return error when listing scanners returns unexpected scanners count", func(t *testing.T) { + mgr := &mocks.Manager{} + scannerManager = mgr + + mgr.On("GetDefault").Return(nil, nil) + mgr.On("List", &q.Query{ + Keywords: map[string]interface{}{"url": "http://trivy:8080"}, + }).Return([]*scanner.Registration{ + {URL: "http://trivy:8080"}, + {URL: "http://trivy:8080"}, + }, nil) + + err := EnsureDefaultScanner("http://trivy:8080") + assert.EqualError(t, err, "expected only one scanner with URL http://trivy:8080 but got 2") + mgr.AssertExpectations(t) + }) + + t.Run("Should set the default scanner when it is not set", func(t *testing.T) { + mgr := &mocks.Manager{} + scannerManager = mgr + + mgr.On("GetDefault").Return(nil, nil) + mgr.On("List", &q.Query{ + Keywords: map[string]interface{}{"url": "http://trivy:8080"}, + }).Return([]*scanner.Registration{ + { + UUID: "trivy-uuid", + URL: "http://trivy:8080", + }, + }, nil) + mgr.On("SetAsDefault", "trivy-uuid").Return(nil) + + err := EnsureDefaultScanner("http://trivy:8080") + assert.NoError(t, err) + mgr.AssertExpectations(t) + }) + + t.Run("Should return error when setting the default scanner fails", func(t *testing.T) { + mgr := &mocks.Manager{} + scannerManager = mgr + + mgr.On("GetDefault").Return(nil, nil) + mgr.On("List", &q.Query{ + Keywords: map[string]interface{}{"url": "http://trivy:8080"}, + }).Return([]*scanner.Registration{ + { + UUID: "trivy-uuid", + URL: "http://trivy:8080", + }, + }, nil) + mgr.On("SetAsDefault", "trivy-uuid").Return(errors.New("DB error")) + + err := EnsureDefaultScanner("http://trivy:8080") + assert.EqualError(t, err, "setting http://trivy:8080 as default scanner: DB error") + mgr.AssertExpectations(t) + }) + +} + +func TestRemoveImmutableScanners(t *testing.T) { + + t.Run("Should do nothing when list of URLs is empty", func(t *testing.T) { + mgr := &mocks.Manager{} + scannerManager = mgr + + err := RemoveImmutableScanners([]string{}) + assert.NoError(t, err) + mgr.AssertExpectations(t) + }) + + t.Run("Should return error when listing scanners fails", func(t *testing.T) { + mgr := &mocks.Manager{} + scannerManager = mgr + + mgr.On("List", &q.Query{ + Keywords: map[string]interface{}{ + "immutable": true, + "ex_url__in": []string{"http://scanner:8080"}, + }, + }).Return(nil, errors.New("DB error")) + + err := RemoveImmutableScanners([]string{"http://scanner:8080"}) + assert.EqualError(t, err, "listing scanners: DB error") + mgr.AssertExpectations(t) + }) + + t.Run("Should delete multiple scanners", func(t *testing.T) { + mgr := &mocks.Manager{} + scannerManager = mgr + + registrations := []*scanner.Registration{ + { + UUID: "uuid-1", + URL: "http://scanner-1", + }, + { + UUID: "uuid-2", + URL: "http://scanner-2", + }} + + mgr.On("List", &q.Query{ + Keywords: map[string]interface{}{ + "immutable": true, + "ex_url__in": []string{ + "http://scanner-1", + "http://scanner-2", + }, + }, + }).Return(registrations, nil) + mgr.On("Delete", "uuid-1").Return(nil) + mgr.On("Delete", "uuid-2").Return(nil) + + err := RemoveImmutableScanners([]string{ + "http://scanner-1", + "http://scanner-2", + }) + assert.NoError(t, err) + mgr.AssertExpectations(t) + }) + + t.Run("Should return error when deleting any scanner fails", func(t *testing.T) { + mgr := &mocks.Manager{} + scannerManager = mgr + + registrations := []*scanner.Registration{ + { + UUID: "uuid-1", + URL: "http://scanner-1", + }, + { + UUID: "uuid-2", + URL: "http://scanner-2", + }} + + mgr.On("List", &q.Query{ + Keywords: map[string]interface{}{ + "immutable": true, + "ex_url__in": []string{ + "http://scanner-1", + "http://scanner-2", + }, + }, + }).Return(registrations, nil) + mgr.On("Delete", "uuid-1").Return(nil) + mgr.On("Delete", "uuid-2").Return(errors.New("DB error")) + + err := RemoveImmutableScanners([]string{ + "http://scanner-1", + "http://scanner-2", + }) + assert.EqualError(t, err, "deleting scanner: uuid-2: DB error") + mgr.AssertExpectations(t) + }) - assert.Nil(EnsureScanner(&scanner.Registration{Name: "Clair", URL: "reg1"})) - assert.Error(EnsureScanner(&scanner.Registration{Name: "Clair", URL: "reg2"})) - assert.Nil(EnsureScanner(&scanner.Registration{Name: "Clair", URL: "reg2"}, true)) }