Update migratrion script (#7728)

* Fix migration script 1. port is string when parsed from configparser 2. remove index and db_user in if condition Signed-off-by: Qian Deng <dengq@vmware.com> * Add port to public_url Add port to public_url Signed-off-by: Qian Deng <dengq@vmware.com> * Customized value for notary and clair db config in notary and clair is hardcoded Signed-off-by: Qian Deng <dengq@vmware.com> * Add notary and clair db config in harbor.yml Add notary clair config to harbor.yml and fix related regression Signed-off-by: Qian Deng <dengq@vmware.com>
2025-01-21 23:21:26 +01:00 · 2019-05-10 10:44:05 +08:00 · 2019-05-10 10:44:05 +08:00 · bb66358df8
commit bb66358df8
parent 4218206b59
14 changed files with 143 additions and 73 deletions
--- a/make/harbor.yml
+++ b/make/harbor.yml
@ -83,11 +83,34 @@ _version: 1.8.0
 # Uncomment external_database if using external database. And the password will replace the the password setting in database.
 # And currently only support postgres.
 # external_database:
-#   host: postgresql
-#   port: 5432
-#   username: postgres
-#   password: root123
-#   ssl_mode: disable
+#   harbor:
+#     host: postgresql
+#     port: 5432
+#     db_name: registry
+#     username: postgres
+#     password: root123
+#     ssl_mode: disable
+#   clair:
+#     host: postgresql
+#     port: 5432
+#     db_name: registry
+#     username: postgres
+#     password: root123
+#     ssl_mode: disable
+#   notary_signer:
+#     host: postgresql
+#     port: 5432
+#     db_name: registry
+#     username: postgres
+#     password: root123
+#     ssl_mode: disable
+#   notary_server:
+#     host: postgresql
+#     port: 5432
+#     db_name: registry
+#     username: postgres
+#     password: root123
+#     ssl_mode: disable

 # Uncomment external_redis if using external Redis server
 # external_redis:
@ -101,4 +124,4 @@ _version: 1.8.0

 # Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
 # uaa:
-#   ca_file: /path/to/ca
+#   ca_file: /path/to/ca
--- a/make/photon/prepare/templates/clair/config.yaml.jinja
+++ b/make/photon/prepare/templates/clair/config.yaml.jinja
@ -2,7 +2,7 @@ clair:
  database:
    type: pgsql
    options:
-      source: postgresql://{{username}}:{{password}}@{{host}}:{{port}}/{{dbname}}?sslmode=disable
+      source: postgresql://{{clair_db_username}}:{{clair_db_password}}@{{clair_db_host}}:{{clair_db_port}}/{{clair_db_name}}?sslmode={{clair_db_sslmode}}

      # Number of elements kept in the cache
      # Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database.
@ -16,7 +16,7 @@ clair:
    # Deadline before an API request will respond with a 503
    timeout: 300s
  updater:
-    interval: {{interval}}h
+    interval: {{clair_updaters_interval}}h

  notifier:
    attempts: 3
--- a/make/photon/prepare/templates/clair/postgres_env.jinja
+++ b/make/photon/prepare/templates/clair/postgres_env.jinja
@ -1 +1 @@
-POSTGRES_PASSWORD={{password}}
+POSTGRES_PASSWORD={{clair_db_password}}
--- a/make/photon/prepare/templates/core/env.jinja
+++ b/make/photon/prepare/templates/core/env.jinja
@ -9,12 +9,12 @@ PORT=8080
 LOG_LEVEL={{log_level}}
 EXT_ENDPOINT={{public_url}}
 DATABASE_TYPE=postgresql
-POSTGRESQL_HOST={{db_host}}
-POSTGRESQL_PORT={{db_port}}
-POSTGRESQL_USERNAME={{db_user}}
-POSTGRESQL_PASSWORD={{db_password}}
-POSTGRESQL_DATABASE=registry
-POSTGRESQL_SSLMODE=disable
+POSTGRESQL_HOST={{harbor_db_host}}
+POSTGRESQL_PORT={{harbor_db_port}}
+POSTGRESQL_USERNAME={{harbor_db_username}}
+POSTGRESQL_PASSWORD={{harbor_db_password}}
+POSTGRESQL_DATABASE={{harbor_db_name}}
+POSTGRESQL_SSLMODE={{harbor_db_sslmode}}
 REGISTRY_URL={{registry_url}}
 TOKEN_SERVICE_URL={{token_service_url}}
 HARBOR_ADMIN_PASSWORD={{harbor_admin_password}}
@ -24,12 +24,12 @@ JOBSERVICE_SECRET={{jobservice_secret}}
 ADMIRAL_URL={{admiral_url}}
 WITH_NOTARY={{with_notary}}
 WITH_CLAIR={{with_clair}}
-CLAIR_DB_PASSWORD={{db_password}}
-CLAIR_DB_HOST={{db_host}}
-CLAIR_DB_PORT={{db_port}}
-CLAIR_DB_USERNAME={{db_user}}
-CLAIR_DB={{clair_db}}
-CLAIR_DB_SSLMODE=disable
+CLAIR_DB_PASSWORD={{clair_db_password}}
+CLAIR_DB_HOST={{clair_db_host}}
+CLAIR_DB_PORT={{clair_db_port}}
+CLAIR_DB_USERNAME={{clair_db_username}}
+CLAIR_DB={{clair_db_name}}
+CLAIR_DB_SSLMODE={{clair_db_sslmode}}
 CORE_URL={{core_url}}
 JOBSERVICE_URL={{jobservice_url}}
 CLAIR_URL={{clair_url}}
--- a/make/photon/prepare/templates/db/env.jinja
+++ b/make/photon/prepare/templates/db/env.jinja
@ -1 +1 @@
-POSTGRES_PASSWORD={{db_password}}
+POSTGRES_PASSWORD={{harbor_db_password}}
--- a/make/photon/prepare/templates/notary/server-config.postgres.json.jinja
+++ b/make/photon/prepare/templates/notary/server-config.postgres.json.jinja
@ -14,7 +14,7 @@
 	},
 	"storage": {
 		"backend": "postgres",
-		"db_url": "postgres://server:password@postgresql:5432/notaryserver?sslmode=disable"
+		"db_url": "postgres://{{notary_server_db_username}}:{{notary_server_db_password}}@{{notary_server_db_host}}:{{notary_server_db_port}}/{{notary_server_db_name}}?sslmode={{notary_server_db_sslmode}}"
    },
    "auth": {
        "type": "token",
--- a/make/photon/prepare/templates/notary/server_env.jinja
+++ b/make/photon/prepare/templates/notary/server_env.jinja
@ -1,2 +1,2 @@
 MIGRATIONS_PATH=migrations/server/postgresql
-DB_URL=postgres://server:password@postgresql:5432/notaryserver?sslmode=disable
+DB_URL=postgres://{{notary_server_db_username}}:{{notary_server_db_password}}@{{notary_server_db_host}}:{{notary_server_db_port}}/{{notary_server_db_name}}?sslmode={{notary_server_db_sslmode}}
--- a/make/photon/prepare/templates/notary/signer-config.postgres.json.jinja
+++ b/make/photon/prepare/templates/notary/signer-config.postgres.json.jinja
@ -9,7 +9,7 @@
 	},
    "storage": {
        "backend": "postgres",
-		    "db_url": "postgres://signer:password@postgresql:5432/notarysigner?sslmode=disable",
-		    "default_alias":"defaultalias"
+			"db_url": "postgres://{{notary_signer_db_username}}:{{notary_signer_db_password}}@{{notary_signer_db_host}}:{{notary_signer_db_port}}/{{notary_signer_db_name}}?sslmode={{notary_signer_db_sslmode}}",
+		    "default_alias": "{{alias}}"
    }
 }  
--- a/make/photon/prepare/templates/notary/signer_env.jinja
+++ b/make/photon/prepare/templates/notary/signer_env.jinja
@ -1,3 +1,3 @@
 NOTARY_SIGNER_DEFAULTALIAS={{alias}}
 MIGRATIONS_PATH=migrations/signer/postgresql
-DB_URL=postgres://signer:password@postgresql:5432/notarysigner?sslmode=disable
+DB_URL=postgres://{{notary_signer_db_username}}:{{notary_signer_db_password}}@{{notary_signer_db_host}}:{{notary_signer_db_port}}/{{notary_signer_db_name}}?sslmode={{notary_signer_db_sslmode}}
--- a/make/photon/prepare/utils/clair.py
+++ b/make/photon/prepare/utils/clair.py
@ -27,19 +27,14 @@ def prepare_clair(config_dict):
    render_jinja(
        postgres_env_template,
        postgres_env_path,
-        password=config_dict['db_password'])
+        **config_dict)

    render_jinja(
        clair_config_template,
        clair_config_path,
        uid=DEFAULT_UID,
        gid=DEFAULT_GID,
-        password= config_dict['db_password'],
-        username= config_dict['db_user'],
-        host= config_dict['db_host'],
-        port= config_dict['db_port'],
-        dbname= config_dict['clair_db'],
-        interval= config_dict['clair_updaters_interval'])
+        **config_dict)

    # config http proxy for Clair
    render_jinja(
--- a/make/photon/prepare/utils/configs.py
+++ b/make/photon/prepare/utils/configs.py
@ -79,16 +79,45 @@ def parse_yaml_config(config_file_path):
        config_dict['cert_path'] = https_config["certificate"]
        config_dict['cert_key_path'] = https_config["private_key"]

-    config_dict['public_url'] = configs.get('external_url') or '{protocol}://{hostname}'.format(**config_dict)
+    if configs.get('external_url'):
+        config_dict['public_url'] = configs.get('external_url')
+    else:
+        if config_dict['protocol'] == 'https':
+            config_dict['public_url'] = '{protocol}://{hostname}:{https_port}'.format(**config_dict)
+        else:
+            config_dict['public_url'] = '{protocol}://{hostname}:{http_port}'.format(**config_dict)

    # DB configs
    db_configs = configs.get('database')
    if db_configs:
-        config_dict['db_host'] = 'postgresql'
-        config_dict['db_port'] = 5432
-        config_dict['db_user'] = 'postgres'
-        config_dict['db_password'] = db_configs.get("password") or ''
-        config_dict['ssl_mode'] = 'disable'
+        # harbor db
+        config_dict['harbor_db_host'] = 'postgresql'
+        config_dict['harbor_db_port'] = 5432
+        config_dict['harbor_db_name'] = 'registry'
+        config_dict['harbor_db_username'] = 'postgres'
+        config_dict['harbor_db_password'] = db_configs.get("password") or ''
+        config_dict['harbor_db_sslmode'] = 'disable'
+        # clari db
+        config_dict['clair_db_host'] = 'postgresql'
+        config_dict['clair_db_port'] = 5432
+        config_dict['clair_db_name'] = 'postgres'
+        config_dict['clair_db_username'] = 'postgres'
+        config_dict['clair_db_password'] = db_configs.get("password") or ''
+        config_dict['clair_db_sslmode'] = 'disable'
+        # notary signer
+        config_dict['notary_signer_db_host'] = 'postgresql'
+        config_dict['notary_signer_db_port'] = 5432
+        config_dict['notary_signer_db_name'] = 'notarysigner'
+        config_dict['notary_signer_db_username'] = 'signer'
+        config_dict['notary_signer_db_password'] = 'password'
+        config_dict['notary_signer_db_sslmode'] = 'disable'
+        # notary server
+        config_dict['notary_server_db_host'] = 'postgresql'
+        config_dict['notary_server_db_port'] = 5432
+        config_dict['notary_server_db_name'] = 'notaryserver'
+        config_dict['notary_server_db_username'] = 'server'
+        config_dict['notary_server_db_password'] = 'password'
+        config_dict['notary_server_db_sslmode'] = 'disable'


    # Data path volume
@ -152,12 +181,34 @@ def parse_yaml_config(config_file_path):
    # external DB, if external_db enabled, it will cover the database config
    external_db_configs = configs.get('external_database') or {}
    if external_db_configs:
-        config_dict['db_password'] = external_db_configs.get('password') or ''
-        config_dict['db_host'] = external_db_configs['host']
-        config_dict['db_port'] = external_db_configs['port']
-        config_dict['db_user'] = external_db_configs['username']
-        if external_db_configs.get('ssl_mode'):
-            config_dict['db_ssl_mode'] = external_db_configs['ssl_mode']
+        # harbor db
+        config_dict['harbor_db_host'] = external_db_configs['harbor']['host']
+        config_dict['harbor_db_port'] = external_db_configs['harbor']['port']
+        config_dict['harbor_db_name'] = external_db_configs['harbor']['db_name']
+        config_dict['harbor_db_username'] = external_db_configs['harbor']['username']
+        config_dict['harbor_db_password'] = external_db_configs['harbor']['password']
+        config_dict['harbor_db_sslmode'] = external_db_configs['harbor']['ssl_mode']
+        # clari db
+        config_dict['clair_db_host'] = external_db_configs['clair']['host']
+        config_dict['clair_db_port'] = external_db_configs['clair']['port']
+        config_dict['clair_db_name'] = external_db_configs['clair']['db_name']
+        config_dict['clair_db_username'] = external_db_configs['clair']['username']
+        config_dict['clair_db_password'] = external_db_configs['clair']['password']
+        config_dict['clair_db_sslmode'] = external_db_configs['clair']['ssl_mode']
+        # notary signer
+        config_dict['notary_signer_db_host'] = external_db_configs['notary_signer']['host']
+        config_dict['notary_signer_db_port'] = external_db_configs['notary_signer']['port']
+        config_dict['notary_signer_db_name'] = external_db_configs['notary_signer']['db_name']
+        config_dict['notary_signer_db_username'] = external_db_configs['notary_signer']['username']
+        config_dict['notary_signer_db_password'] = external_db_configs['notary_signer']['password']
+        config_dict['notary_signer_db_sslmode'] = external_db_configs['notary_signer']['ssl_mode']
+        # notary server
+        config_dict['notary_server_db_host'] = external_db_configs['notary_server']['host']
+        config_dict['notary_server_db_port'] = external_db_configs['notary_server']['port']
+        config_dict['notary_server_db_name'] = external_db_configs['notary_server']['db_name']
+        config_dict['notary_server_db_username'] = external_db_configs['notary_server']['username']
+        config_dict['notary_server_db_password'] = external_db_configs['notary_server']['password']
+        config_dict['notary_server_db_sslmode'] = external_db_configs['notary_server']['ssl_mode']


    # redis config
--- a/make/photon/prepare/utils/db.py
+++ b/make/photon/prepare/utils/db.py
@ -14,7 +14,7 @@ def prepare_db(config_dict):
    render_jinja(
        db_env_template_path,
        db_conf_env,
-        db_password=config_dict['db_password'])
+        harbor_db_password=config_dict['harbor_db_password'])

 def prepare_db_config_dir():
    prepare_config_dir(db_config_dir)
--- a/make/photon/prepare/utils/notary.py
+++ b/make/photon/prepare/utils/notary.py
@ -70,10 +70,6 @@ def prepare_env_notary(nginx_config_dir):
        else:
            raise(Exception("No certs for notary"))

-    # copy server_env to notary config
-    shutil.copy2(
-        os.path.join(notary_template_dir, "server_env.jinja"),
-        os.path.join(notary_config_dir, "server_env"))

    print("Copying nginx configuration file for notary")
    shutil.copy2(
@ -90,32 +86,37 @@ def prepare_notary(config_dict, nginx_config_dir, ssl_cert_path, ssl_cert_key_pa
    prepare_env_notary(nginx_config_dir)

    render_jinja(
-        notary_signer_pg_template,
-        notary_signer_pg_config,
-        uid=DEFAULT_UID,
-        gid=DEFAULT_GID
-        )
+        notary_server_nginx_config_template,
+        os.path.join(nginx_config_dir, "notary.server.conf"),
+        ssl_cert=ssl_cert_path,
+        ssl_cert_key=ssl_cert_key_path)

    render_jinja(
        notary_server_pg_template,
        notary_server_pg_config,
        uid=DEFAULT_UID,
        gid=DEFAULT_GID,
-        token_endpoint=config_dict['public_url'])
-
-    render_jinja(
-        notary_server_nginx_config_template,
-        os.path.join(nginx_config_dir, "notary.server.conf"),
-        ssl_cert=ssl_cert_path,
-        ssl_cert_key=ssl_cert_key_path)
-
-    default_alias = get_alias(secret_key_dir)
-    render_jinja(
-        notary_signer_env_template,
-        notary_signer_env_path,
-        alias=default_alias)
+        token_endpoint=config_dict['public_url'],
+        **config_dict)

    render_jinja(
        notary_server_env_template,
-        notary_server_env_path
-    )
+        notary_server_env_path,
+        **config_dict
+    )
+
+    default_alias = get_alias(secret_key_dir)
+
+    render_jinja(
+        notary_signer_env_template,
+        notary_signer_env_path,
+        alias=default_alias,
+        **config_dict)
+
+    render_jinja(
+        notary_signer_pg_template,
+        notary_signer_pg_config,
+        uid=DEFAULT_UID,
+        gid=DEFAULT_GID,
+        alias=default_alias,
+        **config_dict)
--- a/tools/migration/cfg/migrator_1_8_0/init.py
+++ b/tools/migration/cfg/migrator_1_8_0/init.py
@ -34,7 +34,7 @@ def migrate(input_cfg, output_cfg):
    val = {}
    for k in keys:
        val[k] = d.get(k,'')
-    if val['db_host'] == 'postgresql' and val['db_port'] == 5432 and val['db_user'] == 'postgres':
+    if val['db_host'] == 'postgresql' and val['db_port'] == '5432':
        val['external_db'] = False
    else:
        val['external_db'] = True
@ -46,7 +46,7 @@ def migrate(input_cfg, output_cfg):
            val['registry_storage_provider_name'],
            val['registry_storage_provider_config']
            )
-    if val['redis_host'] == 'redis' and val['redis_port'] == 6379 and not val['redis_password'] and val['redis_db_index'] == '1,2,3':
+    if val['redis_host'] == 'redis' and val['redis_port'] == '6379':
        val['external_redis'] = False
    else:
        val['registry_db_index'], val['jobservice_db_index'], val['chartmuseum_db_index'] = map(int, val['redis_db_index'].split(','))