From bb66358df8bb6af9e022251dbdb0c514b8ea9517 Mon Sep 17 00:00:00 2001 From: Qian Deng Date: Fri, 10 May 2019 10:44:05 +0800 Subject: [PATCH] Update migratrion script (#7728) * Fix migration script 1. port is string when parsed from configparser 2. remove index and db_user in if condition Signed-off-by: Qian Deng * Add port to public_url Add port to public_url Signed-off-by: Qian Deng * Customized value for notary and clair db config in notary and clair is hardcoded Signed-off-by: Qian Deng * Add notary and clair db config in harbor.yml Add notary clair config to harbor.yml and fix related regression Signed-off-by: Qian Deng --- make/harbor.yml | 35 +++++++-- .../prepare/templates/clair/config.yaml.jinja | 4 +- .../templates/clair/postgres_env.jinja | 2 +- make/photon/prepare/templates/core/env.jinja | 24 +++--- make/photon/prepare/templates/db/env.jinja | 2 +- .../notary/server-config.postgres.json.jinja | 2 +- .../prepare/templates/notary/server_env.jinja | 2 +- .../notary/signer-config.postgres.json.jinja | 4 +- .../prepare/templates/notary/signer_env.jinja | 2 +- make/photon/prepare/utils/clair.py | 9 +-- make/photon/prepare/utils/configs.py | 75 ++++++++++++++++--- make/photon/prepare/utils/db.py | 2 +- make/photon/prepare/utils/notary.py | 49 ++++++------ .../migration/cfg/migrator_1_8_0/__init__.py | 4 +- 14 files changed, 143 insertions(+), 73 deletions(-) diff --git a/make/harbor.yml b/make/harbor.yml index 4fb67f982..6bbe1cb51 100644 --- a/make/harbor.yml +++ b/make/harbor.yml @@ -83,11 +83,34 @@ _version: 1.8.0 # Uncomment external_database if using external database. And the password will replace the the password setting in database. # And currently only support postgres. # external_database: -# host: postgresql -# port: 5432 -# username: postgres -# password: root123 -# ssl_mode: disable +# harbor: +# host: postgresql +# port: 5432 +# db_name: registry +# username: postgres +# password: root123 +# ssl_mode: disable +# clair: +# host: postgresql +# port: 5432 +# db_name: registry +# username: postgres +# password: root123 +# ssl_mode: disable +# notary_signer: +# host: postgresql +# port: 5432 +# db_name: registry +# username: postgres +# password: root123 +# ssl_mode: disable +# notary_server: +# host: postgresql +# port: 5432 +# db_name: registry +# username: postgres +# password: root123 +# ssl_mode: disable # Uncomment external_redis if using external Redis server # external_redis: @@ -101,4 +124,4 @@ _version: 1.8.0 # Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert. # uaa: -# ca_file: /path/to/ca \ No newline at end of file +# ca_file: /path/to/ca diff --git a/make/photon/prepare/templates/clair/config.yaml.jinja b/make/photon/prepare/templates/clair/config.yaml.jinja index f626547d8..00062b917 100644 --- a/make/photon/prepare/templates/clair/config.yaml.jinja +++ b/make/photon/prepare/templates/clair/config.yaml.jinja @@ -2,7 +2,7 @@ clair: database: type: pgsql options: - source: postgresql://{{username}}:{{password}}@{{host}}:{{port}}/{{dbname}}?sslmode=disable + source: postgresql://{{clair_db_username}}:{{clair_db_password}}@{{clair_db_host}}:{{clair_db_port}}/{{clair_db_name}}?sslmode={{clair_db_sslmode}} # Number of elements kept in the cache # Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database. @@ -16,7 +16,7 @@ clair: # Deadline before an API request will respond with a 503 timeout: 300s updater: - interval: {{interval}}h + interval: {{clair_updaters_interval}}h notifier: attempts: 3 diff --git a/make/photon/prepare/templates/clair/postgres_env.jinja b/make/photon/prepare/templates/clair/postgres_env.jinja index f38ffa89f..2ce179fb1 100644 --- a/make/photon/prepare/templates/clair/postgres_env.jinja +++ b/make/photon/prepare/templates/clair/postgres_env.jinja @@ -1 +1 @@ -POSTGRES_PASSWORD={{password}} +POSTGRES_PASSWORD={{clair_db_password}} diff --git a/make/photon/prepare/templates/core/env.jinja b/make/photon/prepare/templates/core/env.jinja index 44470dbdb..5e2ae21bb 100644 --- a/make/photon/prepare/templates/core/env.jinja +++ b/make/photon/prepare/templates/core/env.jinja @@ -9,12 +9,12 @@ PORT=8080 LOG_LEVEL={{log_level}} EXT_ENDPOINT={{public_url}} DATABASE_TYPE=postgresql -POSTGRESQL_HOST={{db_host}} -POSTGRESQL_PORT={{db_port}} -POSTGRESQL_USERNAME={{db_user}} -POSTGRESQL_PASSWORD={{db_password}} -POSTGRESQL_DATABASE=registry -POSTGRESQL_SSLMODE=disable +POSTGRESQL_HOST={{harbor_db_host}} +POSTGRESQL_PORT={{harbor_db_port}} +POSTGRESQL_USERNAME={{harbor_db_username}} +POSTGRESQL_PASSWORD={{harbor_db_password}} +POSTGRESQL_DATABASE={{harbor_db_name}} +POSTGRESQL_SSLMODE={{harbor_db_sslmode}} REGISTRY_URL={{registry_url}} TOKEN_SERVICE_URL={{token_service_url}} HARBOR_ADMIN_PASSWORD={{harbor_admin_password}} @@ -24,12 +24,12 @@ JOBSERVICE_SECRET={{jobservice_secret}} ADMIRAL_URL={{admiral_url}} WITH_NOTARY={{with_notary}} WITH_CLAIR={{with_clair}} -CLAIR_DB_PASSWORD={{db_password}} -CLAIR_DB_HOST={{db_host}} -CLAIR_DB_PORT={{db_port}} -CLAIR_DB_USERNAME={{db_user}} -CLAIR_DB={{clair_db}} -CLAIR_DB_SSLMODE=disable +CLAIR_DB_PASSWORD={{clair_db_password}} +CLAIR_DB_HOST={{clair_db_host}} +CLAIR_DB_PORT={{clair_db_port}} +CLAIR_DB_USERNAME={{clair_db_username}} +CLAIR_DB={{clair_db_name}} +CLAIR_DB_SSLMODE={{clair_db_sslmode}} CORE_URL={{core_url}} JOBSERVICE_URL={{jobservice_url}} CLAIR_URL={{clair_url}} diff --git a/make/photon/prepare/templates/db/env.jinja b/make/photon/prepare/templates/db/env.jinja index 9ff165869..9ee8cb797 100644 --- a/make/photon/prepare/templates/db/env.jinja +++ b/make/photon/prepare/templates/db/env.jinja @@ -1 +1 @@ -POSTGRES_PASSWORD={{db_password}} +POSTGRES_PASSWORD={{harbor_db_password}} diff --git a/make/photon/prepare/templates/notary/server-config.postgres.json.jinja b/make/photon/prepare/templates/notary/server-config.postgres.json.jinja index 18c7e5473..46999f4b0 100644 --- a/make/photon/prepare/templates/notary/server-config.postgres.json.jinja +++ b/make/photon/prepare/templates/notary/server-config.postgres.json.jinja @@ -14,7 +14,7 @@ }, "storage": { "backend": "postgres", - "db_url": "postgres://server:password@postgresql:5432/notaryserver?sslmode=disable" + "db_url": "postgres://{{notary_server_db_username}}:{{notary_server_db_password}}@{{notary_server_db_host}}:{{notary_server_db_port}}/{{notary_server_db_name}}?sslmode={{notary_server_db_sslmode}}" }, "auth": { "type": "token", diff --git a/make/photon/prepare/templates/notary/server_env.jinja b/make/photon/prepare/templates/notary/server_env.jinja index 964c00d06..7486b2647 100644 --- a/make/photon/prepare/templates/notary/server_env.jinja +++ b/make/photon/prepare/templates/notary/server_env.jinja @@ -1,2 +1,2 @@ MIGRATIONS_PATH=migrations/server/postgresql -DB_URL=postgres://server:password@postgresql:5432/notaryserver?sslmode=disable +DB_URL=postgres://{{notary_server_db_username}}:{{notary_server_db_password}}@{{notary_server_db_host}}:{{notary_server_db_port}}/{{notary_server_db_name}}?sslmode={{notary_server_db_sslmode}} diff --git a/make/photon/prepare/templates/notary/signer-config.postgres.json.jinja b/make/photon/prepare/templates/notary/signer-config.postgres.json.jinja index 6c5189c63..ea5cf5079 100644 --- a/make/photon/prepare/templates/notary/signer-config.postgres.json.jinja +++ b/make/photon/prepare/templates/notary/signer-config.postgres.json.jinja @@ -9,7 +9,7 @@ }, "storage": { "backend": "postgres", - "db_url": "postgres://signer:password@postgresql:5432/notarysigner?sslmode=disable", - "default_alias":"defaultalias" + "db_url": "postgres://{{notary_signer_db_username}}:{{notary_signer_db_password}}@{{notary_signer_db_host}}:{{notary_signer_db_port}}/{{notary_signer_db_name}}?sslmode={{notary_signer_db_sslmode}}", + "default_alias": "{{alias}}" } } \ No newline at end of file diff --git a/make/photon/prepare/templates/notary/signer_env.jinja b/make/photon/prepare/templates/notary/signer_env.jinja index 60857f2bd..2482b5a0b 100644 --- a/make/photon/prepare/templates/notary/signer_env.jinja +++ b/make/photon/prepare/templates/notary/signer_env.jinja @@ -1,3 +1,3 @@ NOTARY_SIGNER_DEFAULTALIAS={{alias}} MIGRATIONS_PATH=migrations/signer/postgresql -DB_URL=postgres://signer:password@postgresql:5432/notarysigner?sslmode=disable +DB_URL=postgres://{{notary_signer_db_username}}:{{notary_signer_db_password}}@{{notary_signer_db_host}}:{{notary_signer_db_port}}/{{notary_signer_db_name}}?sslmode={{notary_signer_db_sslmode}} diff --git a/make/photon/prepare/utils/clair.py b/make/photon/prepare/utils/clair.py index 86f49abd3..72db85038 100644 --- a/make/photon/prepare/utils/clair.py +++ b/make/photon/prepare/utils/clair.py @@ -27,19 +27,14 @@ def prepare_clair(config_dict): render_jinja( postgres_env_template, postgres_env_path, - password=config_dict['db_password']) + **config_dict) render_jinja( clair_config_template, clair_config_path, uid=DEFAULT_UID, gid=DEFAULT_GID, - password= config_dict['db_password'], - username= config_dict['db_user'], - host= config_dict['db_host'], - port= config_dict['db_port'], - dbname= config_dict['clair_db'], - interval= config_dict['clair_updaters_interval']) + **config_dict) # config http proxy for Clair render_jinja( diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index 36a71601b..4790c0030 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -79,16 +79,45 @@ def parse_yaml_config(config_file_path): config_dict['cert_path'] = https_config["certificate"] config_dict['cert_key_path'] = https_config["private_key"] - config_dict['public_url'] = configs.get('external_url') or '{protocol}://{hostname}'.format(**config_dict) + if configs.get('external_url'): + config_dict['public_url'] = configs.get('external_url') + else: + if config_dict['protocol'] == 'https': + config_dict['public_url'] = '{protocol}://{hostname}:{https_port}'.format(**config_dict) + else: + config_dict['public_url'] = '{protocol}://{hostname}:{http_port}'.format(**config_dict) # DB configs db_configs = configs.get('database') if db_configs: - config_dict['db_host'] = 'postgresql' - config_dict['db_port'] = 5432 - config_dict['db_user'] = 'postgres' - config_dict['db_password'] = db_configs.get("password") or '' - config_dict['ssl_mode'] = 'disable' + # harbor db + config_dict['harbor_db_host'] = 'postgresql' + config_dict['harbor_db_port'] = 5432 + config_dict['harbor_db_name'] = 'registry' + config_dict['harbor_db_username'] = 'postgres' + config_dict['harbor_db_password'] = db_configs.get("password") or '' + config_dict['harbor_db_sslmode'] = 'disable' + # clari db + config_dict['clair_db_host'] = 'postgresql' + config_dict['clair_db_port'] = 5432 + config_dict['clair_db_name'] = 'postgres' + config_dict['clair_db_username'] = 'postgres' + config_dict['clair_db_password'] = db_configs.get("password") or '' + config_dict['clair_db_sslmode'] = 'disable' + # notary signer + config_dict['notary_signer_db_host'] = 'postgresql' + config_dict['notary_signer_db_port'] = 5432 + config_dict['notary_signer_db_name'] = 'notarysigner' + config_dict['notary_signer_db_username'] = 'signer' + config_dict['notary_signer_db_password'] = 'password' + config_dict['notary_signer_db_sslmode'] = 'disable' + # notary server + config_dict['notary_server_db_host'] = 'postgresql' + config_dict['notary_server_db_port'] = 5432 + config_dict['notary_server_db_name'] = 'notaryserver' + config_dict['notary_server_db_username'] = 'server' + config_dict['notary_server_db_password'] = 'password' + config_dict['notary_server_db_sslmode'] = 'disable' # Data path volume @@ -152,12 +181,34 @@ def parse_yaml_config(config_file_path): # external DB, if external_db enabled, it will cover the database config external_db_configs = configs.get('external_database') or {} if external_db_configs: - config_dict['db_password'] = external_db_configs.get('password') or '' - config_dict['db_host'] = external_db_configs['host'] - config_dict['db_port'] = external_db_configs['port'] - config_dict['db_user'] = external_db_configs['username'] - if external_db_configs.get('ssl_mode'): - config_dict['db_ssl_mode'] = external_db_configs['ssl_mode'] + # harbor db + config_dict['harbor_db_host'] = external_db_configs['harbor']['host'] + config_dict['harbor_db_port'] = external_db_configs['harbor']['port'] + config_dict['harbor_db_name'] = external_db_configs['harbor']['db_name'] + config_dict['harbor_db_username'] = external_db_configs['harbor']['username'] + config_dict['harbor_db_password'] = external_db_configs['harbor']['password'] + config_dict['harbor_db_sslmode'] = external_db_configs['harbor']['ssl_mode'] + # clari db + config_dict['clair_db_host'] = external_db_configs['clair']['host'] + config_dict['clair_db_port'] = external_db_configs['clair']['port'] + config_dict['clair_db_name'] = external_db_configs['clair']['db_name'] + config_dict['clair_db_username'] = external_db_configs['clair']['username'] + config_dict['clair_db_password'] = external_db_configs['clair']['password'] + config_dict['clair_db_sslmode'] = external_db_configs['clair']['ssl_mode'] + # notary signer + config_dict['notary_signer_db_host'] = external_db_configs['notary_signer']['host'] + config_dict['notary_signer_db_port'] = external_db_configs['notary_signer']['port'] + config_dict['notary_signer_db_name'] = external_db_configs['notary_signer']['db_name'] + config_dict['notary_signer_db_username'] = external_db_configs['notary_signer']['username'] + config_dict['notary_signer_db_password'] = external_db_configs['notary_signer']['password'] + config_dict['notary_signer_db_sslmode'] = external_db_configs['notary_signer']['ssl_mode'] + # notary server + config_dict['notary_server_db_host'] = external_db_configs['notary_server']['host'] + config_dict['notary_server_db_port'] = external_db_configs['notary_server']['port'] + config_dict['notary_server_db_name'] = external_db_configs['notary_server']['db_name'] + config_dict['notary_server_db_username'] = external_db_configs['notary_server']['username'] + config_dict['notary_server_db_password'] = external_db_configs['notary_server']['password'] + config_dict['notary_server_db_sslmode'] = external_db_configs['notary_server']['ssl_mode'] # redis config diff --git a/make/photon/prepare/utils/db.py b/make/photon/prepare/utils/db.py index 49f4b7e1f..53ef3d93e 100644 --- a/make/photon/prepare/utils/db.py +++ b/make/photon/prepare/utils/db.py @@ -14,7 +14,7 @@ def prepare_db(config_dict): render_jinja( db_env_template_path, db_conf_env, - db_password=config_dict['db_password']) + harbor_db_password=config_dict['harbor_db_password']) def prepare_db_config_dir(): prepare_config_dir(db_config_dir) \ No newline at end of file diff --git a/make/photon/prepare/utils/notary.py b/make/photon/prepare/utils/notary.py index db791ea64..8d1d1175e 100644 --- a/make/photon/prepare/utils/notary.py +++ b/make/photon/prepare/utils/notary.py @@ -70,10 +70,6 @@ def prepare_env_notary(nginx_config_dir): else: raise(Exception("No certs for notary")) - # copy server_env to notary config - shutil.copy2( - os.path.join(notary_template_dir, "server_env.jinja"), - os.path.join(notary_config_dir, "server_env")) print("Copying nginx configuration file for notary") shutil.copy2( @@ -90,32 +86,37 @@ def prepare_notary(config_dict, nginx_config_dir, ssl_cert_path, ssl_cert_key_pa prepare_env_notary(nginx_config_dir) render_jinja( - notary_signer_pg_template, - notary_signer_pg_config, - uid=DEFAULT_UID, - gid=DEFAULT_GID - ) + notary_server_nginx_config_template, + os.path.join(nginx_config_dir, "notary.server.conf"), + ssl_cert=ssl_cert_path, + ssl_cert_key=ssl_cert_key_path) render_jinja( notary_server_pg_template, notary_server_pg_config, uid=DEFAULT_UID, gid=DEFAULT_GID, - token_endpoint=config_dict['public_url']) - - render_jinja( - notary_server_nginx_config_template, - os.path.join(nginx_config_dir, "notary.server.conf"), - ssl_cert=ssl_cert_path, - ssl_cert_key=ssl_cert_key_path) - - default_alias = get_alias(secret_key_dir) - render_jinja( - notary_signer_env_template, - notary_signer_env_path, - alias=default_alias) + token_endpoint=config_dict['public_url'], + **config_dict) render_jinja( notary_server_env_template, - notary_server_env_path - ) \ No newline at end of file + notary_server_env_path, + **config_dict + ) + + default_alias = get_alias(secret_key_dir) + + render_jinja( + notary_signer_env_template, + notary_signer_env_path, + alias=default_alias, + **config_dict) + + render_jinja( + notary_signer_pg_template, + notary_signer_pg_config, + uid=DEFAULT_UID, + gid=DEFAULT_GID, + alias=default_alias, + **config_dict) diff --git a/tools/migration/cfg/migrator_1_8_0/__init__.py b/tools/migration/cfg/migrator_1_8_0/__init__.py index 0266db09c..1c9336b23 100644 --- a/tools/migration/cfg/migrator_1_8_0/__init__.py +++ b/tools/migration/cfg/migrator_1_8_0/__init__.py @@ -34,7 +34,7 @@ def migrate(input_cfg, output_cfg): val = {} for k in keys: val[k] = d.get(k,'') - if val['db_host'] == 'postgresql' and val['db_port'] == 5432 and val['db_user'] == 'postgres': + if val['db_host'] == 'postgresql' and val['db_port'] == '5432': val['external_db'] = False else: val['external_db'] = True @@ -46,7 +46,7 @@ def migrate(input_cfg, output_cfg): val['registry_storage_provider_name'], val['registry_storage_provider_config'] ) - if val['redis_host'] == 'redis' and val['redis_port'] == 6379 and not val['redis_password'] and val['redis_db_index'] == '1,2,3': + if val['redis_host'] == 'redis' and val['redis_port'] == '6379': val['external_redis'] = False else: val['registry_db_index'], val['jobservice_db_index'], val['chartmuseum_db_index'] = map(int, val['redis_db_index'].split(','))