From fe69a5df99ef151dc330947d9f86cbbf0ecedf87 Mon Sep 17 00:00:00 2001 From: He Weiwei Date: Wed, 13 Nov 2019 02:35:21 +0000 Subject: [PATCH] build(scanner-adapter): bump up clair adapter to v1.0.1-rc2 Signed-off-by: He Weiwei --- Makefile | 2 +- make/harbor.yml | 1 + .../prepare/templates/clair-adapter/env.jinja | 4 +- .../docker_compose/docker-compose.yml.jinja | 12 +- make/photon/prepare/utils/configs.py | 110 +++++++++++++----- .../cfg/migrator_1_10_0/harbor.yml.jinja | 2 + 6 files changed, 98 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index c01aeeec2..0276fb9c0 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ CLAIRVERSION=v2.1.0 MIGRATORVERSION=$(VERSIONTAG) REDISVERSION=$(VERSIONTAG) NOTARYMIGRATEVERSION=v3.5.4 -CLAIRADAPTERVERSION=v1.0.0 +CLAIRADAPTERVERSION=v1.0.1-rc2 # version of chartmuseum CHARTMUSEUMVERSION=v0.9.0 diff --git a/make/harbor.yml b/make/harbor.yml index c16d73769..86ee42493 100644 --- a/make/harbor.yml +++ b/make/harbor.yml @@ -140,6 +140,7 @@ _version: 1.10.0 # registry_db_index: 1 # jobservice_db_index: 2 # chartmuseum_db_index: 3 +# clair_db_index: 4 # Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert. # uaa: diff --git a/make/photon/prepare/templates/clair-adapter/env.jinja b/make/photon/prepare/templates/clair-adapter/env.jinja index f75a0f006..906832128 100644 --- a/make/photon/prepare/templates/clair-adapter/env.jinja +++ b/make/photon/prepare/templates/clair-adapter/env.jinja @@ -1,2 +1,4 @@ SCANNER_LOG_LEVEL={{log_level}} -SCANNER_CLAIR_URL={{clair_url}} \ No newline at end of file +SCANNER_CLAIR_URL={{clair_url}} +SCANNER_CLAIR_DATABASE_URL=postgresql://{{clair_db_username}}:{{clair_db_password}}@{{clair_db_host}}:{{clair_db_port}}/{{clair_db_name}}?sslmode={{clair_db_sslmode}} +SCANNER_STORE_REDIS_URL={{redis_url_clair}} \ No newline at end of file diff --git a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja index 359a1550f..c132db764 100644 --- a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja +++ b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja @@ -258,8 +258,13 @@ services: harbor: {% if with_chartmuseum %} harbor-chartmuseum: - aliases: - - redis + aliases: + - redis + {% endif %} + {% if with_clair %} + harbor-clair: + aliases: + - redis {% endif %} dns_search: . depends_on: @@ -422,6 +427,9 @@ services: dns_search: . depends_on: - clair +{% if external_redis == False %} + - redis +{% endif %} logging: driver: "syslog" options: diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index 12beb07d5..3e74167cb 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -303,35 +303,8 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_chartmuseu else: config_dict['external_database'] = False - - # redis config - redis_configs = configs.get("external_redis") - if redis_configs: - config_dict['external_redis'] = True - # using external_redis - config_dict['redis_host'] = redis_configs['host'] - config_dict['redis_port'] = redis_configs['port'] - config_dict['redis_password'] = redis_configs.get("password") or '' - config_dict['redis_db_index_reg'] = redis_configs.get('registry_db_index') or 1 - config_dict['redis_db_index_js'] = redis_configs.get('jobservice_db_index') or 2 - config_dict['redis_db_index_chart'] = redis_configs.get('chartmuseum_db_index') or 3 - else: - config_dict['external_redis'] = False - ## Using local redis - config_dict['redis_host'] = 'redis' - config_dict['redis_port'] = 6379 - config_dict['redis_password'] = '' - config_dict['redis_db_index_reg'] = 1 - config_dict['redis_db_index_js'] = 2 - config_dict['redis_db_index_chart'] = 3 - - # redis://[arbitrary_username:password@]ipaddress:port/database_index - if config_dict.get('redis_password'): - config_dict['redis_url_js'] = "redis://anonymous:%s@%s:%s/%s" % (config_dict['redis_password'], config_dict['redis_host'], config_dict['redis_port'], config_dict['redis_db_index_js']) - config_dict['redis_url_reg'] = "redis://anonymous:%s@%s:%s/%s" % (config_dict['redis_password'], config_dict['redis_host'], config_dict['redis_port'], config_dict['redis_db_index_reg']) - else: - config_dict['redis_url_js'] = "redis://%s:%s/%s" % (config_dict['redis_host'], config_dict['redis_port'], config_dict['redis_db_index_js']) - config_dict['redis_url_reg'] = "redis://%s:%s/%s" % (config_dict['redis_host'], config_dict['redis_port'], config_dict['redis_db_index_reg']) + # update redis configs + config_dict.update(get_redis_configs(configs.get("external_redis", None), with_clair)) # auto generated secret string for core config_dict['core_secret'] = generate_random_string(16) @@ -343,3 +316,82 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_chartmuseu config_dict['uaa'] = configs.get('uaa') or {} return config_dict + + +def get_redis_url(db, redis=None): + """Returns redis url with format `redis://[arbitrary_username:password@]ipaddress:port/database_index` + + >>> get_redis_url(1) + 'redis://redis:6379/1' + >>> get_redis_url(1, {'host': 'localhost', 'password': 'password'}) + 'redis://anonymous:password@localhost:6379/1' + """ + kwargs = { + 'host': 'redis', + 'port': 6379, + 'password': '', + } + kwargs.update(redis or {}) + kwargs['db'] = db + + if kwargs['password']: + return "redis://anonymous:{password}@{host}:{port}/{db}".format(**kwargs) + return "redis://{host}:{port}/{db}".format(**kwargs) + + +def get_redis_configs(external_redis=None, with_clair=True): + """Returns configs for redis + + >>> get_redis_configs()['external_redis'] + False + >>> get_redis_configs()['redis_url_reg'] + 'redis://redis:6379/1' + >>> get_redis_configs()['redis_url_js'] + 'redis://redis:6379/2' + >>> get_redis_configs()['redis_url_clair'] + 'redis://redis:6379/4' + + >>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['external_redis'] + True + >>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_reg'] + 'redis://anonymous:pass@localhost:6379/1' + >>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_js'] + 'redis://anonymous:pass@localhost:6379/2' + >>> get_redis_configs({'host': 'localhost', 'password': 'pass'})['redis_url_clair'] + 'redis://anonymous:pass@localhost:6379/4' + + >>> 'redis_url_clair' not in get_redis_configs(with_clair=False) + True + """ + + configs = dict(external_redis=bool(external_redis)) + + # internal redis config as the default + redis = { + 'host': 'redis', + 'port': 6379, + 'password': '', + 'registry_db_index': 1, + 'jobservice_db_index': 2, + 'chartmuseum_db_index': 3, + 'clair_db_index': 4, + } + + # overwriting existing keys by external_redis + redis.update(external_redis or {}) + + configs['redis_host'] = redis['host'] + configs['redis_port'] = redis['port'] + configs['redis_password'] = redis['password'] + configs['redis_db_index_reg'] = redis['registry_db_index'] + configs['redis_db_index_js'] = redis['jobservice_db_index'] + configs['redis_db_index_chart'] = redis['chartmuseum_db_index'] + + configs['redis_url_js'] = get_redis_url(configs['redis_db_index_js'], redis) + configs['redis_url_reg'] = get_redis_url(configs['redis_db_index_reg'], redis) + + if with_clair: + configs['redis_db_index_clair'] = redis['clair_db_index'] + configs['redis_url_clair'] = get_redis_url(configs['redis_db_index_clair'], redis) + + return configs diff --git a/tools/migration/cfg/migrator_1_10_0/harbor.yml.jinja b/tools/migration/cfg/migrator_1_10_0/harbor.yml.jinja index c9386104e..5b0c65109 100644 --- a/tools/migration/cfg/migrator_1_10_0/harbor.yml.jinja +++ b/tools/migration/cfg/migrator_1_10_0/harbor.yml.jinja @@ -286,6 +286,7 @@ external_redis: registry_db_index: {{ external_redis.registry_db_index }} jobservice_db_index: {{ external_redis.jobservice_db_index }} chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }} + clair_db_index: 4 {% else %} # Umcomments external_redis if using external Redis server # external_redis: @@ -296,6 +297,7 @@ external_redis: # registry_db_index: 1 # jobservice_db_index: 2 # chartmuseum_db_index: 3 +# clair_db_index: 4 {% endif %} {% if uaa is defined %}