Add migratior script

Add migrator template and script

Signed-off-by: Qian Deng <dengq@vmware.com>

make/photon/prepare/utils/configs.py

@@ -147,12 +147,12 @@ def parse_yaml_config(config_file_path):
     # Clair configs
     clair_configs = configs.get("clair")
     if clair_configs:
-        config_dict['clair_db_password'] = clair_configs.get("clair_db_password")
-        config_dict['clair_db_host'] = clair_configs.get("clair_db_host")
-        config_dict['clair_db_port'] = clair_configs.get("clair_db_port")
-        config_dict['clair_db_username'] = clair_configs.get("clair_db_username")
-        config_dict['clair_db'] = clair_configs.get("clair_db")
-        config_dict['clair_updaters_interval'] = clair_configs.get("clair_updaters_interval")
+        config_dict['clair_db_password'] = clair_configs.get("db_password")
+        config_dict['clair_db_host'] = clair_configs.get("db_host")
+        config_dict['clair_db_port'] = clair_configs.get("db_port")
+        config_dict['clair_db_username'] = clair_configs.get("db_username")
+        config_dict['clair_db'] = clair_configs.get("db")
+        config_dict['clair_updaters_interval'] = clair_configs.get("updaters_interval")
         config_dict['clair_http_proxy'] = clair_configs.get('http_proxy')
         config_dict['clair_https_proxy'] = clair_configs.get('https_proxy')
         config_dict['clair_no_proxy'] = clair_configs.get('no_proxy')
@@ -183,10 +183,10 @@ def parse_yaml_config(config_file_path):
     # Redis configs
     redis_configs = configs.get("redis")
     if redis_configs:
-        config_dict['redis_host'] = redis_configs.get("redis_host") or ''
-        config_dict['redis_port'] = redis_configs.get("redis_port") or ''
-        config_dict['redis_password'] = redis_configs.get("redis_password") or ''
-        config_dict['redis_db_index'] = redis_configs.get("redis_db_index") or ''
+        config_dict['redis_host'] = redis_configs.get("host") or ''
+        config_dict['redis_port'] = redis_configs.get("port") or ''
+        config_dict['redis_password'] = redis_configs.get("password") or ''
+        config_dict['redis_db_index'] = redis_configs.get("db_index") or ''
         db_indexs = config_dict['redis_db_index'].split(',')
         config_dict['redis_db_index_reg'] = db_indexs[0]
         config_dict['redis_db_index_js'] = db_indexs[1]

make/prepare

@@ -4,7 +4,7 @@ host_make_path="$( cd "$(dirname "$0")" ; pwd -P )"
 
 echo host make path is set to ${host_make_path}
 data_path=$(grep '^[^#]*data_volume' ${host_make_path}/harbor.yml | awk '{print $NF}')
-log_path=$(grep '^[^#]*log_location' ${host_make_path}/harbor.yml | awk '{print $NF}')
+log_path=$(grep '^[^#]*location' ${host_make_path}/harbor.yml | awk '{print $NF}')
 secretkey_path=$(grep '^[^#]*secretkey_path' ${host_make_path}/harbor.yml | awk '{print $NF}')
 
 docker run -it --rm -v ${host_make_path}:/harbor_make -v $data_path:/data -v $log_path:/var/log/harbor -v $secretkey_path:$secretkey_path goharbor/prepare:1.7.1 $@

tools/migration/cfg/migrator_1_8_0/__init__.py

@@ -0,0 +1,42 @@
+from __future__ import print_function
+import utils
+import os
+acceptable_versions = ['1.7.0']
+keys = [
+    'hostname',
+    'ui_url_protocol',
+    'customize_crt',
+    'ssl_cert',
+    'ssl_cert_key',
+    'secretkey_path',
+    'admiral_url',
+    'log_rotate_count',
+    'log_rotate_size',
+    'http_proxy',
+    'https_proxy',
+    'no_proxy',
+    'db_host',
+    'db_password',
+    'db_port',
+    'db_user',
+    'clair_db_host',
+    'clair_db_password',
+    'clair_db_port',
+    'clair_db_username',
+    'clair_db',
+    'uaa_endpoint',
+    'uaa_clientid',
+    'uaa_clientsecret',
+    'uaa_verify_cert',
+    'uaa_ca_cert',
+    'registry_storage_provider_name',
+    'registry_storage_provider_config'
+    ]
+
+def migrate(input_cfg, output_cfg):
+    d = utils.read_conf(input_cfg)
+    val = {}
+    for k in keys:
+        val[k] = d.get(k,'')
+    tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.yml.tpl')
+    utils.render(tpl_path, output_cfg, **val)

tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl

@@ -0,0 +1,133 @@
+## Configuration file of Harbor
+
+#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
+_version: 1.7.0
+#The IP address or hostname to access admin UI and registry service.
+#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
+#DO NOT comment out this line, modify the value of "hostname" directly, or the installation will fail.
+hostname: $hostname
+
+#The protocol for accessing the UI and token/notification service, by default it is http.
+#It can be set to https if ssl is enabled on nginx.
+ui_url_protocol: $ui_url_protocol
+
+#Maximum number of job workers in job service  
+max_job_workers: 10
+
+#Determine whether or not to generate certificate for the registry's token.
+#If the value is on, the prepare script creates new root cert and private key 
+#for generating token to access the registry. If the value is off the default key/cert will be used.
+#This flag also controls the creation of the notary signer's cert.
+customize_crt: $customize_crt
+
+# The default data volume
+data_volume: /data
+
+#The path of cert and key files for nginx, they are applied only the protocol is set to https
+ssl_cert: $ssl_cert
+ssl_cert_key: $ssl_cert_key
+
+#The path of secretkey storage
+secretkey_path: $secretkey_path
+
+#Admiral's url, comment this attribute, or set its value to NA when Harbor is standalone
+admiral_url: $admiral_url
+
+log:
+  #Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
+  rotate_count: $log_rotate_count
+  #Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. 
+  #If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G 
+  #are all valid.
+  rotate_size: $log_rotate_size
+
+  # The directory that store log files
+  location: /var/log/harbor
+
+#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES
+#only take effect in the first boot, the subsequent changes of these properties 
+#should be performed on web ui
+
+#************************BEGIN INITIAL PROPERTIES************************
+
+##By default the auth mode is db_auth, i.e. the credentials are stored in a local database.
+#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server.
+auth_mode: db_auth
+
+#The base DN from which to look up a user in LDAP/AD
+ldap_basedn: ou=people,dc=mydomain,dc=com
+
+#The attribute used to name a LDAP/AD group, it could be cn, name
+ldap_group_gid: cn
+
+#The following attributes only need to be set when auth mode is uaa_auth
+uaa_ca_cert: $uaa_ca_cert
+
+#The flag to control what users have permission to create projects
+#The default value "everyone" allows everyone to creates a project. 
+#Set to "adminonly" so that only admin user can create project.
+project_creation_restriction: everyone
+
+#************************END INITIAL PROPERTIES************************
+
+
+
+database:
+  #The address of the Harbor database. Only need to change when using external db.
+  host: $db_host
+  #The port of Harbor database host
+  port: $db_port
+  #The user name of Harbor database
+  username: $db_user
+  #The password for the root user of Harbor DB. Change this before any production use.
+  password: $db_password
+
+redis:
+  # Redis connection address
+  redis_host: redis
+  # Redis connection port
+  redis_port: 6379
+  # Redis connection password
+  redis_password:
+  # Redis connection db index
+  # db_index 1,2,3 is for registry, jobservice and chartmuseum. 
+  # db_index 0 is for UI, it's unchangeable
+  redis_db_index: 1,2,3
+
+clair:
+  #Clair DB host address. Only change it when using an exteral DB.
+  db_host: $clair_db_host
+  #The password of the Clair's postgres database. Only effective when Harbor is deployed with Clair.
+  #Please update it before deployment. Subsequent update will cause Clair's API server and Harbor unable to access Clair's database.
+  db_password: $clair_db_password
+  #Clair DB connect port
+  db_port: $clair_db_port
+  #Clair DB username
+  db_username: $clair_db_username
+  #Clair default database
+  db: $clair_db
+  #The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
+  updaters_interval: 12
+  #Config http proxy for Clair, e.g. http://my.proxy.com:3128
+  #Clair doesn't need to connect to harbor internal components via http proxy.
+  http_proxy: $http_proxy
+  https_proxy: $https_proxy
+  no_proxy: $no_proxy
+
+storage:
+  #Please be aware that the following storage settings will be applied to both docker registry and helm chart repository.
+  #registry_storage_provider can be: filesystem, s3, gcs, azure, etc.
+  registry_storage_provider_name: $registry_storage_provider_name
+  #registry_storage_provider_config is a comma separated "key: value" pairs, e.g. "key1: value, key2: value2".
+  #To avoid duplicated configurations, both docker registry and chart repository follow the same storage configuration specifications of docker registry.
+  #Refer to https://docs.docker.com/registry/configuration/#storage for all available configuration.
+  registry_storage_provider_config: $registry_storage_provider_config
+  #registry_custom_ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
+  #of registry's and chart repository's containers.  This is usually needed when the user hosts a internal storage with self signed certificate.
+  registry_custom_ca_bundle:
+
+#If reload_config=true, all settings which present in harbor.yml take effect after prepare and restart harbor, it overwrites exsiting settings.
+#reload_config=true
+#Regular expression to match skipped environment variables
+#skip_reload_env_pattern: (^EMAIL.*)|(^LDAP.*)
+