From 46fb43bc25eb1e74dd2dea12c4646a06607016ed Mon Sep 17 00:00:00 2001 From: Daniel Pacak Date: Thu, 12 Mar 2020 09:37:41 +0100 Subject: [PATCH] chore: Bump up Trivy adapter to v0.4.0 Allows configuring SCANNER_TRIVY_GITHUB_TOKEN environment variable, which is passed to trivy executable binary when it starts scanning a given artifact. This is to increase GitHub requests rate limit from 60 per hours (for anonymous requests) to 5000 when Trivy download its vulnerabilities database. Signed-off-by: Daniel Pacak --- Makefile | 4 ++-- make/harbor.yml.tmpl | 20 +++++++++++++++++++ .../prepare/templates/trivy-adapter/env.jinja | 5 +++-- make/photon/prepare/utils/configs.py | 13 ++++++++---- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 55076e51c..a0cc5863e 100644 --- a/Makefile +++ b/Makefile @@ -102,8 +102,8 @@ NOTARYVERSION=v0.6.1 CLAIRVERSION=v2.1.1 NOTARYMIGRATEVERSION=v3.5.4 CLAIRADAPTERVERSION=v1.0.1 -TRIVYVERSION=v0.4.3 -TRIVYADAPTERVERSION=v0.2.3 +TRIVYVERSION=v0.5.2 +TRIVYADAPTERVERSION=v0.4.0 # version of chartmuseum CHARTMUSEUMVERSION=v0.9.0 diff --git a/make/harbor.yml.tmpl b/make/harbor.yml.tmpl index 997e1edda..cf1fc583d 100644 --- a/make/harbor.yml.tmpl +++ b/make/harbor.yml.tmpl @@ -59,6 +59,26 @@ clair: # The interval of clair updaters, the unit is hour, set to 0 to disable the updaters. updaters_interval: 12 +# Trivy configuration +trivy: + # github_token The GitHub access token to download Trivy DB + # + # Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases. + # It is downloaded by Trivy from the GitHub release page https://github.com/aquasecurity/trivy-db/releases and cached + # in the local file system (/home/scanner/.cache/trivy/db/trivy.db). In addition, the database contains the update + # timestamp so Trivy can detect whether it should download a newer version from the Internet or use the cached one. + # Currently, the database is updated every 12 hours and published as a new release to GitHub. + # + # Anonymous downloads from GitHub are subject to the limit of 60 requests per hour. Normally such rate limit is enough + # for production operations. If, for any reason, it's not enough, you could increase the rate limit to 5000 + # requests per hour by specifying the GitHub access token. For more details on GitHub rate limiting please consult + # https://developer.github.com/v3/#rate-limiting + # + # You can create a GitHub token by following the instuctions in + # https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line + # + # github_token: xxx + jobservice: # Maximum number of job workers in job service max_job_workers: 10 diff --git a/make/photon/prepare/templates/trivy-adapter/env.jinja b/make/photon/prepare/templates/trivy-adapter/env.jinja index 9bdb20e93..2df6758aa 100644 --- a/make/photon/prepare/templates/trivy-adapter/env.jinja +++ b/make/photon/prepare/templates/trivy-adapter/env.jinja @@ -1,10 +1,11 @@ SCANNER_LOG_LEVEL={{log_level}} -SCANNER_STORE_REDIS_URL={{redis_url_trivy}} +SCANNER_STORE_REDIS_URL={{trivy_redis_url}} SCANNER_STORE_REDIS_NAMESPACE=harbor.scanner.trivy:store -SCANNER_JOB_QUEUE_REDIS_URL={{redis_url_trivy}} +SCANNER_JOB_QUEUE_REDIS_URL={{trivy_redis_url}} SCANNER_JOB_QUEUE_REDIS_NAMESPACE=harbor.scanner.trivy:job-queue SCANNER_TRIVY_CACHE_DIR=/home/scanner/.cache/trivy SCANNER_TRIVY_REPORTS_DIR=/home/scanner/.cache/reports SCANNER_TRIVY_VULN_TYPE=os,library SCANNER_TRIVY_SEVERITY=UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL SCANNER_TRIVY_IGNORE_UNFIXED=false +SCANNER_TRIVY_GITHUB_TOKEN={{trivy_github_token}} diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index 90e8d4fa9..877301446 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -239,6 +239,11 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit updaters_interval = clair_configs.get("updaters_interval", None) config_dict['clair_updaters_interval'] = 12 if updaters_interval is None else updaters_interval + # Trivy configs, optional + trivy_configs = configs.get("trivy") or {} + trivy_github_token = trivy_configs.get("github_token") or '' + config_dict['trivy_github_token'] = trivy_github_token + # Chart configs chart_configs = configs.get("chart") or {} config_dict['chart_absolute_url'] = chart_configs.get('absolute_url') or '' @@ -363,7 +368,7 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True): 'redis://redis:6379/2' >>> get_redis_configs()['redis_url_clair'] 'redis://redis:6379/4' - >>> get_redis_configs()['redis_url_trivy'] + >>> get_redis_configs()['trivy_redis_url'] 'redis://redis:6379/5' >>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['external_redis'] @@ -374,12 +379,12 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True): 'redis://anonymous:pass@localhost:6379/2' >>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_clair'] 'redis://anonymous:pass@localhost:6379/4' - >>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_trivy'] + >>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['trivy_redis_url'] 'redis://anonymous:pass@localhost:6379/5' >>> 'redis_url_clair' not in get_redis_configs(with_clair=False) True - >>> 'redis_url_trivy' not in get_redis_configs(with_trivy=False) + >>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False) True """ @@ -418,6 +423,6 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True): if with_trivy: configs['redis_db_index_trivy'] = redis['trivy_db_index'] - configs['redis_url_trivy'] = get_redis_url(configs['redis_db_index_trivy'], redis) + configs['trivy_redis_url'] = get_redis_url(configs['redis_db_index_trivy'], redis) return configs