From c1e676ad9921e90aed759f80aa4784427a49de0f Mon Sep 17 00:00:00 2001 From: Qian Deng Date: Tue, 9 Apr 2019 15:51:51 +0800 Subject: [PATCH] Add migration script from 1.7.0 to 1.8.0 Add jinja2 to migrator Add template to migrator Add config upgrading script Signed-off-by: Qian Deng --- make/harbor.yml | 2 +- tools/migration/Dockerfile | 2 +- .../migration/cfg/migrator_1_8_0/__init__.py | 47 +++++--- .../cfg/migrator_1_8_0/harbor.yml.jinja | 113 +++++++++++++++++ .../cfg/migrator_1_8_0/harbor.yml.tpl | 114 ------------------ tools/migration/cfg/run.py | 3 +- tools/migration/cfg/utils.py | 22 ++++ 7 files changed, 170 insertions(+), 133 deletions(-) create mode 100644 tools/migration/cfg/migrator_1_8_0/harbor.yml.jinja delete mode 100644 tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl diff --git a/make/harbor.yml b/make/harbor.yml index 7abb44217..1a3f78e37 100644 --- a/make/harbor.yml +++ b/make/harbor.yml @@ -71,7 +71,7 @@ log: location: /var/log/harbor #This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! -_version: 1.7.0 +_version: 1.8.0 # Uncomment external_database if using external database. And the password will replace the the password setting in database. # And currently ontly support postgres. diff --git a/tools/migration/Dockerfile b/tools/migration/Dockerfile index 4bebce16c..222ceb057 100644 --- a/tools/migration/Dockerfile +++ b/tools/migration/Dockerfile @@ -7,7 +7,7 @@ RUN tdnf distro-sync -y \ && tdnf remove -y toybox \ && tdnf install -y sed shadow procps-ng gawk gzip sudo net-tools glibc-i18n >> /dev/null\ && groupadd -r -g 10000 mysql && useradd --no-log-init -r -g 10000 -u 10000 mysql \ - && tdnf install -y mariadb-server mariadb mariadb-devel python2 python2-devel python-pip gcc \ + && tdnf install -y mariadb-server mariadb mariadb-devel python2 python2-devel python-pip gcc PyYAML python-jinja2\ linux-api-headers glibc-devel binutils zlib-devel openssl-devel postgresql python-psycopg2 >> /dev/null \ && pip install mysqlclient alembic \ && mkdir /docker-entrypoint-initdb.d /docker-entrypoint-updatedb.d \ diff --git a/tools/migration/cfg/migrator_1_8_0/__init__.py b/tools/migration/cfg/migrator_1_8_0/__init__.py index db26f8de1..0266db09c 100644 --- a/tools/migration/cfg/migrator_1_8_0/__init__.py +++ b/tools/migration/cfg/migrator_1_8_0/__init__.py @@ -1,14 +1,13 @@ from __future__ import print_function import utils import os +from jinja2 import Environment, FileSystemLoader acceptable_versions = ['1.7.0'] keys = [ 'hostname', 'ui_url_protocol', - 'customize_crt', 'ssl_cert', 'ssl_cert_key', - 'secretkey_path', 'admiral_url', 'log_rotate_count', 'log_rotate_size', @@ -19,18 +18,15 @@ keys = [ 'db_password', 'db_port', 'db_user', - 'clair_db_host', - 'clair_db_password', - 'clair_db_port', - 'clair_db_username', - 'clair_db', - 'uaa_endpoint', - 'uaa_clientid', - 'uaa_clientsecret', - 'uaa_verify_cert', - 'uaa_ca_cert', + 'redis_host', + 'redis_port', + 'redis_password', + 'redis_db_index', + 'clair_updaters_interval', + 'max_job_workers', 'registry_storage_provider_name', - 'registry_storage_provider_config' + 'registry_storage_provider_config', + 'registry_custom_ca_bundle' ] def migrate(input_cfg, output_cfg): @@ -38,5 +34,26 @@ def migrate(input_cfg, output_cfg): val = {} for k in keys: val[k] = d.get(k,'') - tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.yml.tpl') - utils.render(tpl_path, output_cfg, **val) + if val['db_host'] == 'postgresql' and val['db_port'] == 5432 and val['db_user'] == 'postgres': + val['external_db'] = False + else: + val['external_db'] = True + # If using default filesystem, didn't need registry_storage_provider_config config + if val['registry_storage_provider_name'] == 'filesystem' and not val.get('registry_storage_provider_config'): + val['storage_provider_info'] = '' + else: + val['storage_provider_info'] = utils.get_storage_provider_info( + val['registry_storage_provider_name'], + val['registry_storage_provider_config'] + ) + if val['redis_host'] == 'redis' and val['redis_port'] == 6379 and not val['redis_password'] and val['redis_db_index'] == '1,2,3': + val['external_redis'] = False + else: + val['registry_db_index'], val['jobservice_db_index'], val['chartmuseum_db_index'] = map(int, val['redis_db_index'].split(',')) + val['external_redis'] = True + + this_dir = os.path.dirname(__file__) + tpl = Environment(loader=FileSystemLoader(this_dir)).get_template('harbor.yml.jinja') + + with open(output_cfg, 'w') as f: + f.write(tpl.render(**val)) \ No newline at end of file diff --git a/tools/migration/cfg/migrator_1_8_0/harbor.yml.jinja b/tools/migration/cfg/migrator_1_8_0/harbor.yml.jinja new file mode 100644 index 000000000..e70c3ba06 --- /dev/null +++ b/tools/migration/cfg/migrator_1_8_0/harbor.yml.jinja @@ -0,0 +1,113 @@ +## Configuration file of Harbor + +#The IP address or hostname to access admin UI and registry service. +#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. +hostname: {{hostname}} + +# http related comfig +http: + # port for http, default is 80. If https enabled, this port will redirect to https port + port: 80 + +{% if ui_url_protocol == 'https'%} +https: + port: 443 + #The path of cert and key files for nginx + certificate: {{ ssl_cert }} + private_key: {{ ssl_cert_key }} +{% else %} +# https: +# port: 443 +# #The path of cert and key files for nginx +# certificate: /your/certificate/path +# private_key: /your/private/key/path +{% endif %} + +# Uncomment extearnal_url if you want to enable external proxy +# And when it enabled the hostname will no longger used +# external_url: https://reg.mydomain.com:8433 + +# The initial password of Harbor admin +# It only works in first time to install harbor +# Remember Change the admin password from UI after launching Harbor. +harbor_admin_password: Harbor12345 + +## Harbor DB configuration +database: + #The password for the root user of Harbor DB. Change this before any production use. + password: {{ db_password }} + +# The default data volume +data_volume: /data + +# Harbor Storage settings by default is using data_volume dir on local filesystem +# Uncomment storage_service setting If you want to using external storage +storage_service: + # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore + # of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signed certificate. + ca_bundle: {{ registry_custom_ca_bundle }} + + {{storage_provider_info}} + +# Clair configuration +clair: + # The interval of clair updaters, the unit is hour, set to 0 to disable the updaters. + updaters_interval: {{ clair_updaters_interval }} + + # Config http proxy for Clair, e.g. http://my.proxy.com:3128 + # Clair doesn't need to connect to harbor internal components via http proxy. + http_proxy: {{ http_proxy }} + https_proxy: {{ https_proxy }} + no_proxy: {{ no_proxy }} + +jobservice: + # Maximum number of job workers in job service + max_job_workers: {{ max_job_workers }} + +# Log configurations +log: + # options are debug, info, warn, error + level: info + # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. + rotate_count: {{ log_rotate_count }} + # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. + # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G + # are all valid. + rotate_size: {{ log_rotate_size }} + # The directory on your host that store log + location: /var/log/harbor + +#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! +_version: 1.8.0 + +{% if external_db %} +# Uncomment external_database if using external database. And the password will replace the the password setting in database. +# And currently ontly support postgres. +external_database: + host: {{ db_host }} + port: {{ db_port }} + username: {{ db_user }} + password: {{ db_password }} + ssl_mode: disable +{% endif %} + +{% if external_redis %} +external_redis: + host: {{ redis_host }} + port: {{ redis_port }} + password: {{ redis_password }} + # db_index 0 is for core, it's unchangeable + registry_db_index: {{ registry_db_index }} + jobservice_db_index: {{ jobservice_db_index }} + chartmuseum_db_index: {{ chartmuseum_db_index }} +{% else %} +# Umcomments external_redis if using external Redis server +# external_redis: +# host: redis +# port: 6379 +# password: +# # db_index 0 is for core, it's unchangeable +# registry_db_index: 1 +# jobservice_db_index: 2 +# chartmuseum_db_index: 3 +{% endif %} \ No newline at end of file diff --git a/tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl b/tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl deleted file mode 100644 index 7716c5fe1..000000000 --- a/tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl +++ /dev/null @@ -1,114 +0,0 @@ -## Configuration file of Harbor - -#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! -_version: 1.7.0 -#The IP address or hostname to access admin UI and registry service. -#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. -#DO NOT comment out this line, modify the value of "hostname" directly, or the installation will fail. -hostname: $hostname - -#The protocol for accessing the UI and token/notification service, by default it is http. -#It can be set to https if ssl is enabled on nginx. -ui_url_protocol: $ui_url_protocol - -#Maximum number of job workers in job service -max_job_workers: 10 - -#Determine whether or not to generate certificate for the registry's token. -#If the value is on, the prepare script creates new root cert and private key -#for generating token to access the registry. If the value is off the default key/cert will be used. -#This flag also controls the creation of the notary signer's cert. -customize_crt: $customize_crt - -# The default data volume -data_volume: /data - -#The path of cert and key files for nginx, they are applied only the protocol is set to https -ssl_cert: $ssl_cert -ssl_cert_key: $ssl_cert_key - -#The path of secretkey storage -secretkey_path: $secretkey_path - -#Admiral's url, comment this attribute, or set its value to NA when Harbor is standalone -admiral_url: $admiral_url - -log: - #Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. - rotate_count: $log_rotate_count - #Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. - #If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G - #are all valid. - rotate_size: $log_rotate_size - - # The directory that store log files - location: /var/log/harbor - -#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES -#only take effect in the first boot, the subsequent changes of these properties -#should be performed on web ui - -##The initial password of Harbor admin, only works for the first time when Harbor starts. -#It has no effect after the first launch of Harbor. -#Change the admin password from UI after launching Harbor. -harbor_admin_password: Harbor12345 - -database: - #The address of the Harbor database. Only need to change when using external db. - host: $db_host - #The port of Harbor database host - port: $db_port - #The user name of Harbor database - username: $db_user - #The password for the root user of Harbor DB. Change this before any production use. - password: $db_password - -redis: - # Redis connection address - redis_host: redis - # Redis connection port - redis_port: 6379 - # Redis connection password - redis_password: - # Redis connection db index - # db_index 1,2,3 is for registry, jobservice and chartmuseum. - # db_index 0 is for UI, it's unchangeable - redis_db_index: 1,2,3 - -clair: - #Clair DB host address. Only change it when using an exteral DB. - db_host: $clair_db_host - #The password of the Clair's postgres database. Only effective when Harbor is deployed with Clair. - #Please update it before deployment. Subsequent update will cause Clair's API server and Harbor unable to access Clair's database. - db_password: $clair_db_password - #Clair DB connect port - db_port: $clair_db_port - #Clair DB username - db_username: $clair_db_username - #Clair default database - db: $clair_db - #The interval of clair updaters, the unit is hour, set to 0 to disable the updaters. - updaters_interval: 12 - #Config http proxy for Clair, e.g. http://my.proxy.com:3128 - #Clair doesn't need to connect to harbor internal components via http proxy. - http_proxy: $http_proxy - https_proxy: $https_proxy - no_proxy: $no_proxy - -storage: - #Please be aware that the following storage settings will be applied to both docker registry and helm chart repository. - #registry_storage_provider can be: filesystem, s3, gcs, azure, etc. - registry_storage_provider_name: $registry_storage_provider_name - #registry_storage_provider_config is a comma separated "key: value" pairs, e.g. "key1: value, key2: value2". - #To avoid duplicated configurations, both docker registry and chart repository follow the same storage configuration specifications of docker registry. - #Refer to https://docs.docker.com/registry/configuration/#storage for all available configuration. - registry_storage_provider_config: $registry_storage_provider_config - #registry_custom_ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore - #of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signed certificate. - registry_custom_ca_bundle: - -#If reload_config=true, all settings which present in harbor.yml take effect after prepare and restart harbor, it overwrites exsiting settings. -#reload_config=true -#Regular expression to match skipped environment variables -#skip_reload_env_pattern: (^EMAIL.*)|(^LDAP.*) - diff --git a/tools/migration/cfg/run.py b/tools/migration/cfg/run.py index b49c4e9d9..7a03030fd 100644 --- a/tools/migration/cfg/run.py +++ b/tools/migration/cfg/run.py @@ -13,7 +13,7 @@ import shutil import sys def main(): - target_version = '1.7.0' + target_version = '1.8.0' parser = argparse.ArgumentParser(description='migrator of harbor.cfg') parser.add_argument('--input', '-i', action="store", dest='input_path', required=True, help='The path to the old harbor.cfg that provides input value, this required value') parser.add_argument('--output','-o', action="store", dest='output_path', required=False, help='The path of the migrated harbor.cfg, if not set the input file will be overwritten') @@ -62,7 +62,6 @@ def search(basedir, input_ver, target_ver, l): l.append(target_ver) return True return False - if __name__ == "__main__": main() diff --git a/tools/migration/cfg/utils.py b/tools/migration/cfg/utils.py index 7791b7422..078cb1b40 100644 --- a/tools/migration/cfg/utils.py +++ b/tools/migration/cfg/utils.py @@ -43,3 +43,25 @@ def render(src, dest, **kw): t = Template(open(src, 'r').read()) with open(dest, 'w') as f: f.write(t.substitute(**kw)) + +def get_storage_provider_info(provider_name, provider_config): + provider_config = provider_config.strip('" ') + if not provider_config.strip(" "): + return '' + + storage_provider_cfg_map = {} + for k_v in provider_config.split(","): + if k_v > 0: + kvs = k_v.split(": ") # add space suffix to avoid existing ":" in the value + if len(kvs) == 2: + #key must not be empty + if kvs[0].strip() != "": + storage_provider_cfg_map[kvs[0].strip()] = kvs[1].strip() + + # generate storage configuration section in yaml format + + storage_provider_conf_list = [provider_name + ':'] + for config in storage_provider_cfg_map.items(): + storage_provider_conf_list.append('{}: {}'.format(*config)) + storage_provider_info = ('\n' + ' ' * 4).join(storage_provider_conf_list) + return storage_provider_info