Add migration script from 1.7.0 to 1.8.0

Add jinja2 to migrator
Add template to migrator
Add config upgrading script

Signed-off-by: Qian Deng <dengq@vmware.com>

make/harbor.yml

@@ -71,7 +71,7 @@ log:
   location: /var/log/harbor
 
 #This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
-_version: 1.7.0
+_version: 1.8.0
 
 # Uncomment external_database if using external database. And the password will replace the the password setting in database.
 # And currently ontly support postgres.

tools/migration/Dockerfile

@@ -7,7 +7,7 @@ RUN tdnf distro-sync -y \
     && tdnf remove -y toybox \
     && tdnf install -y sed shadow procps-ng gawk gzip sudo net-tools glibc-i18n >> /dev/null\
     && groupadd -r -g 10000 mysql && useradd --no-log-init -r -g 10000 -u 10000 mysql \
-    && tdnf install -y mariadb-server mariadb mariadb-devel python2 python2-devel python-pip gcc \
+    && tdnf install -y mariadb-server mariadb mariadb-devel python2 python2-devel python-pip gcc PyYAML python-jinja2\
     linux-api-headers glibc-devel binutils zlib-devel openssl-devel postgresql python-psycopg2 >> /dev/null \
     && pip install mysqlclient alembic \
     && mkdir /docker-entrypoint-initdb.d /docker-entrypoint-updatedb.d \

tools/migration/cfg/migrator_1_8_0/__init__.py

@@ -1,14 +1,13 @@
 from __future__ import print_function
 import utils
 import os
+from jinja2 import Environment, FileSystemLoader
 acceptable_versions = ['1.7.0']
 keys = [
     'hostname',
     'ui_url_protocol',
-    'customize_crt',
     'ssl_cert',
     'ssl_cert_key',
-    'secretkey_path',
     'admiral_url',
     'log_rotate_count',
     'log_rotate_size',
@@ -19,18 +18,15 @@ keys = [
     'db_password',
     'db_port',
     'db_user',
-    'clair_db_host',
-    'clair_db_password',
-    'clair_db_port',
-    'clair_db_username',
-    'clair_db',
-    'uaa_endpoint',
-    'uaa_clientid',
-    'uaa_clientsecret',
-    'uaa_verify_cert',
-    'uaa_ca_cert',
+    'redis_host',
+    'redis_port',
+    'redis_password',
+    'redis_db_index',
+    'clair_updaters_interval',
+    'max_job_workers',
     'registry_storage_provider_name',
-    'registry_storage_provider_config'
+    'registry_storage_provider_config',
+    'registry_custom_ca_bundle'
     ]
 
 def migrate(input_cfg, output_cfg):
@@ -38,5 +34,26 @@ def migrate(input_cfg, output_cfg):
     val = {}
     for k in keys:
         val[k] = d.get(k,'')
-    tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.yml.tpl')
-    utils.render(tpl_path, output_cfg, **val)
+    if val['db_host'] == 'postgresql' and val['db_port'] == 5432 and val['db_user'] == 'postgres':
+        val['external_db'] = False
+    else:
+        val['external_db'] = True
+    # If using default filesystem, didn't need registry_storage_provider_config config
+    if val['registry_storage_provider_name'] == 'filesystem' and not val.get('registry_storage_provider_config'):
+        val['storage_provider_info'] = ''
+    else:
+        val['storage_provider_info'] = utils.get_storage_provider_info(
+            val['registry_storage_provider_name'],
+            val['registry_storage_provider_config']
+            )
+    if val['redis_host'] == 'redis' and val['redis_port'] == 6379 and not val['redis_password'] and val['redis_db_index'] == '1,2,3':
+        val['external_redis'] = False
+    else:
+        val['registry_db_index'], val['jobservice_db_index'], val['chartmuseum_db_index'] = map(int, val['redis_db_index'].split(','))
+        val['external_redis'] = True
+
+    this_dir = os.path.dirname(__file__)
+    tpl = Environment(loader=FileSystemLoader(this_dir)).get_template('harbor.yml.jinja')
+
+    with open(output_cfg, 'w') as f:
+        f.write(tpl.render(**val))

tools/migration/cfg/migrator_1_8_0/harbor.yml.jinja

@@ -0,0 +1,113 @@
+## Configuration file of Harbor
+
+#The IP address or hostname to access admin UI and registry service.
+#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
+hostname: {{hostname}}
+
+# http related comfig
+http:
+  # port for http, default is 80. If https enabled, this port will redirect to https port
+  port: 80
+
+{% if ui_url_protocol == 'https'%}
+https:
+  port: 443
+  #The path of cert and key files for nginx
+  certificate: {{ ssl_cert }}
+  private_key: {{ ssl_cert_key }}
+{% else %}
+# https:
+#   port: 443
+#   #The path of cert and key files for nginx
+#   certificate: /your/certificate/path
+#   private_key: /your/private/key/path
+{% endif %}
+
+# Uncomment extearnal_url if you want to enable external proxy
+# And when it enabled the hostname will no longger used
+# external_url: https://reg.mydomain.com:8433
+
+# The initial password of Harbor admin
+# It only works in first time to install harbor
+# Remember Change the admin password from UI after launching Harbor.
+harbor_admin_password: Harbor12345
+
+## Harbor DB configuration
+database:
+  #The password for the root user of Harbor DB. Change this before any production use.
+  password: {{ db_password }}
+
+# The default data volume
+data_volume: /data
+
+# Harbor Storage settings by default is using data_volume dir on local filesystem
+# Uncomment storage_service setting If you want to using external storage
+storage_service:
+  # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
+  # of registry's and chart repository's containers.  This is usually needed when the user hosts a internal storage with self signed certificate.
+  ca_bundle: {{ registry_custom_ca_bundle }}
+
+  {{storage_provider_info}}
+
+# Clair configuration
+clair:
+  # The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
+  updaters_interval: {{ clair_updaters_interval }}
+
+  # Config http proxy for Clair, e.g. http://my.proxy.com:3128
+  # Clair doesn't need to connect to harbor internal components via http proxy.
+  http_proxy: {{ http_proxy }}
+  https_proxy: {{ https_proxy }}
+  no_proxy: {{ no_proxy }}
+
+jobservice:
+  # Maximum number of job workers in job service  
+  max_job_workers: {{ max_job_workers }}
+
+# Log configurations
+log:
+  # options are debug, info, warn, error
+  level: info
+  # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
+  rotate_count: {{ log_rotate_count }}
+  # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. 
+  # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G 
+  # are all valid.
+  rotate_size: {{ log_rotate_size }}
+  # The directory on your host that store log
+  location: /var/log/harbor
+
+#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
+_version: 1.8.0
+
+{% if external_db %}
+# Uncomment external_database if using external database. And the password will replace the the password setting in database.
+# And currently ontly support postgres.
+external_database:
+  host: {{ db_host }}
+  port: {{ db_port }}
+  username: {{ db_user }}
+  password: {{ db_password }}
+  ssl_mode: disable
+{% endif %}
+
+{% if external_redis %}
+external_redis:
+  host: {{ redis_host }}
+  port: {{ redis_port }}
+  password: {{ redis_password }}
+  # db_index 0 is for core, it's unchangeable
+  registry_db_index: {{ registry_db_index }}
+  jobservice_db_index: {{ jobservice_db_index }}
+  chartmuseum_db_index: {{ chartmuseum_db_index }}
+{% else %}
+# Umcomments external_redis if using external Redis server
+# external_redis:
+#   host: redis
+#   port: 6379
+#   password:
+#   # db_index 0 is for core, it's unchangeable
+#   registry_db_index: 1
+#   jobservice_db_index: 2
+#   chartmuseum_db_index: 3
+{% endif %}

tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl

@@ -1,114 +0,0 @@
-## Configuration file of Harbor
-
-#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
-_version: 1.7.0
-#The IP address or hostname to access admin UI and registry service.
-#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
-#DO NOT comment out this line, modify the value of "hostname" directly, or the installation will fail.
-hostname: $hostname
-
-#The protocol for accessing the UI and token/notification service, by default it is http.
-#It can be set to https if ssl is enabled on nginx.
-ui_url_protocol: $ui_url_protocol
-
-#Maximum number of job workers in job service  
-max_job_workers: 10
-
-#Determine whether or not to generate certificate for the registry's token.
-#If the value is on, the prepare script creates new root cert and private key 
-#for generating token to access the registry. If the value is off the default key/cert will be used.
-#This flag also controls the creation of the notary signer's cert.
-customize_crt: $customize_crt
-
-# The default data volume
-data_volume: /data
-
-#The path of cert and key files for nginx, they are applied only the protocol is set to https
-ssl_cert: $ssl_cert
-ssl_cert_key: $ssl_cert_key
-
-#The path of secretkey storage
-secretkey_path: $secretkey_path
-
-#Admiral's url, comment this attribute, or set its value to NA when Harbor is standalone
-admiral_url: $admiral_url
-
-log:
-  #Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
-  rotate_count: $log_rotate_count
-  #Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. 
-  #If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G 
-  #are all valid.
-  rotate_size: $log_rotate_size
-
-  # The directory that store log files
-  location: /var/log/harbor
-
-#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES
-#only take effect in the first boot, the subsequent changes of these properties 
-#should be performed on web ui
-
-##The initial password of Harbor admin, only works for the first time when Harbor starts.
-#It has no effect after the first launch of Harbor.
-#Change the admin password from UI after launching Harbor.
-harbor_admin_password: Harbor12345
-
-database:
-  #The address of the Harbor database. Only need to change when using external db.
-  host: $db_host
-  #The port of Harbor database host
-  port: $db_port
-  #The user name of Harbor database
-  username: $db_user
-  #The password for the root user of Harbor DB. Change this before any production use.
-  password: $db_password
-
-redis:
-  # Redis connection address
-  redis_host: redis
-  # Redis connection port
-  redis_port: 6379
-  # Redis connection password
-  redis_password:
-  # Redis connection db index
-  # db_index 1,2,3 is for registry, jobservice and chartmuseum. 
-  # db_index 0 is for UI, it's unchangeable
-  redis_db_index: 1,2,3
-
-clair:
-  #Clair DB host address. Only change it when using an exteral DB.
-  db_host: $clair_db_host
-  #The password of the Clair's postgres database. Only effective when Harbor is deployed with Clair.
-  #Please update it before deployment. Subsequent update will cause Clair's API server and Harbor unable to access Clair's database.
-  db_password: $clair_db_password
-  #Clair DB connect port
-  db_port: $clair_db_port
-  #Clair DB username
-  db_username: $clair_db_username
-  #Clair default database
-  db: $clair_db
-  #The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
-  updaters_interval: 12
-  #Config http proxy for Clair, e.g. http://my.proxy.com:3128
-  #Clair doesn't need to connect to harbor internal components via http proxy.
-  http_proxy: $http_proxy
-  https_proxy: $https_proxy
-  no_proxy: $no_proxy
-
-storage:
-  #Please be aware that the following storage settings will be applied to both docker registry and helm chart repository.
-  #registry_storage_provider can be: filesystem, s3, gcs, azure, etc.
-  registry_storage_provider_name: $registry_storage_provider_name
-  #registry_storage_provider_config is a comma separated "key: value" pairs, e.g. "key1: value, key2: value2".
-  #To avoid duplicated configurations, both docker registry and chart repository follow the same storage configuration specifications of docker registry.
-  #Refer to https://docs.docker.com/registry/configuration/#storage for all available configuration.
-  registry_storage_provider_config: $registry_storage_provider_config
-  #registry_custom_ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
-  #of registry's and chart repository's containers.  This is usually needed when the user hosts a internal storage with self signed certificate.
-  registry_custom_ca_bundle:
-
-#If reload_config=true, all settings which present in harbor.yml take effect after prepare and restart harbor, it overwrites exsiting settings.
-#reload_config=true
-#Regular expression to match skipped environment variables
-#skip_reload_env_pattern: (^EMAIL.*)|(^LDAP.*)
-

tools/migration/cfg/run.py

@@ -13,7 +13,7 @@ import shutil
 import sys
 
 def main():
-    target_version = '1.7.0'
+    target_version = '1.8.0'
     parser = argparse.ArgumentParser(description='migrator of harbor.cfg') 
     parser.add_argument('--input', '-i', action="store", dest='input_path', required=True, help='The path to the old harbor.cfg that provides input value, this required value')
     parser.add_argument('--output','-o', action="store", dest='output_path', required=False, help='The path of the migrated harbor.cfg, if not set the input file will be overwritten')
@@ -62,7 +62,6 @@ def search(basedir, input_ver, target_ver, l):
                 l.append(target_ver)
                 return True
     return False
-        
 
 if __name__ == "__main__":
     main()

tools/migration/cfg/utils.py

@@ -43,3 +43,25 @@ def render(src, dest, **kw):
     t = Template(open(src, 'r').read())
     with open(dest, 'w') as f:
         f.write(t.substitute(**kw))
+
+def get_storage_provider_info(provider_name, provider_config):
+    provider_config = provider_config.strip('" ')
+    if not provider_config.strip(" "):
+        return ''
+
+    storage_provider_cfg_map = {}
+    for k_v in provider_config.split(","):
+        if k_v > 0:
+            kvs = k_v.split(": ") # add space suffix to avoid existing ":" in the value
+            if len(kvs) == 2:
+                #key must not be empty
+                if kvs[0].strip() != "":
+                    storage_provider_cfg_map[kvs[0].strip()] = kvs[1].strip()
+
+    # generate storage configuration section in yaml format
+
+    storage_provider_conf_list = [provider_name + ':']
+    for config in storage_provider_cfg_map.items():
+        storage_provider_conf_list.append('{}: {}'.format(*config))
+    storage_provider_info = ('\n' + ' ' * 4).join(storage_provider_conf_list)
+    return storage_provider_info