Merge pull request #4423 from reasonerjt/config-migration

Provide migration scripts for harbor.cfg

make/harbor.cfg

@@ -1,5 +1,7 @@
 ## Configuration file of Harbor
 
+#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
+_version = 1.5.0
 #The IP address or hostname to access admin UI and registry service.
 #DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
 hostname = reg.mydomain.com

tools/migration/cfg/migrate.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+from __future__ import print_function
+import argparse
+import os
+import utils
+import importlib
+import glob
+import shutil
+
+def main():
+    target_version = '1.5.0'
+    parser = argparse.ArgumentParser(description='migrator of harbor.cfg') 
+    parser.add_argument('--input', '-i', action="store", dest='input_path', required=True, help='The path to the old harbor.cfg that provides input value, this required value')
+    parser.add_argument('--output','-o', action="store", dest='output_path', required=False, help='The path of the migrated harbor.cfg, if not set the input file will be overwritten')
+    parser.add_argument('--target', action="store", dest='target_version', help='The target version that the harbor.cfg will be migrated to.')
+    args = parser.parse_args()
+    if args.output_path is None:
+        args.output_path = args.input_path
+    if args.target_version is not None:
+        target_version = args.target_version
+    input_version =  utils.get_conf_version(args.input_path)
+    curr_dir = os.path.dirname(__file__)
+    chain = []
+    if not search(curr_dir, input_version, target_version, chain):
+        print ("No migrator for version: %s" % input_version)
+        os.exit(1)
+    else:
+        print ("input version: %s, migrator chain: %s" % (input_version, chain))
+    curr_input_path = args.input_path
+    for c in chain:
+    #TODO: more real-world testing needed for chained-migration.
+        m = importlib.import_module(to_module_path(c))
+        curr_output_path = "harbor.cfg.%s.tmp" % c
+        print("migrating to version %s" % c)
+        m.migrate(curr_input_path, curr_output_path)
+        curr_input_path = curr_output_path
+    shutil.copy(curr_output_path, args.output_path)
+    print("Written new values to %s" % args.output_path)
+    for tmp_f in glob.glob("harbor.cfg.*.tmp"):
+        os.remove(tmp_f)
+
+def to_module_path(ver):
+    return "migrator_%s" % ver.replace(".","_")
+
+def search(basedir, input_ver, target_ver, l):
+    module = to_module_path(target_ver)
+    if os.path.isdir(os.path.join(basedir, module)):
+        m = importlib.import_module(module)
+        if input_ver in m.acceptable_versions:
+            l.append(target_ver)
+            return True
+        for v in m.acceptable_versions:
+            if search(basedir, input_ver, v, l):
+                l.append(target_ver)
+                return True
+    return False
+        
+
+if __name__ == "__main__":
+    main()

tools/migration/cfg/migrator_1_5_0/__init__.py

@@ -0,0 +1,41 @@
+from __future__ import print_function
+import utils
+import os
+acceptable_versions = ['1.2.0', '1.3.0', '1.4.0']
+
+#In 1.5 template the placeholder has the same value as the attribute name.
+default = {
+    'log_rotate_count':'50',
+    'log_rotate_size':'200M',
+    'db_host':'mysql',
+    'db_port':'3306',
+    'db_user':'root',
+    'redis_url':'',
+    'clair_db_host':'postgres',
+    'clair_db_port':'5432',
+    'clair_db_username':'postgres',
+    'clair_db':'postgres',
+    'uaa_endpoint':'uaa.mydomain.org',
+    'uaa_clientid':'id',
+    'uaa_clientsecret':'secret',
+    'uaa_verify_cert':'true',
+    'uaa_ca_cert':'/path/to/ca.pem',
+    'registry_storage_provider_name':'filesystem',
+    'registry_storage_provider_config':''
+}
+
+def migrate(input_cfg, output_cfg):
+    d = utils.read_conf(input_cfg)
+    keys = default.keys()
+    keys.extend(['hostname', 'ui_url_protocol', 'max_job_workers', 'customize_crt', 
+            'ssl_cert', 'ssl_cert_key', 'secretkey_path', 'admiral_url', 'db_password', 'clair_db_password'])
+    val = {}
+    for k in keys:
+        if d.has_key(k):
+            val[k] = d[k]
+        else:
+            val[k] = default[k]
+    tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.cfg.tpl')
+    utils.render(tpl_path, output_cfg, **val)
+
+

tools/migration/cfg/migrator_1_5_0/harbor.cfg.tpl

@@ -0,0 +1,161 @@
+## Configuration file of Harbor
+
+#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
+_version = 1.5.0
+#The IP address or hostname to access admin UI and registry service.
+#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
+hostname = $hostname
+
+#The protocol for accessing the UI and token/notification service, by default it is http.
+#It can be set to https if ssl is enabled on nginx.
+ui_url_protocol = $ui_url_protocol
+
+#Maximum number of job workers in job service  
+max_job_workers = $max_job_workers
+
+#Determine whether or not to generate certificate for the registry's token.
+#If the value is on, the prepare script creates new root cert and private key 
+#for generating token to access the registry. If the value is off the default key/cert will be used.
+#This flag also controls the creation of the notary signer's cert.
+customize_crt = $customize_crt
+
+#The path of cert and key files for nginx, they are applied only the protocol is set to https
+ssl_cert = $ssl_cert
+ssl_cert_key = $ssl_cert_key
+
+#The path of secretkey storage
+secretkey_path = $secretkey_path
+
+#Admiral's url, comment this attribute, or set its value to NA when Harbor is standalone
+admiral_url = $admiral_url
+
+#Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
+log_rotate_count = $log_rotate_count
+#Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. 
+#If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G 
+#are all valid.
+log_rotate_size = $log_rotate_size
+
+#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES
+#only take effect in the first boot, the subsequent changes of these properties 
+#should be performed on web ui
+
+#************************BEGIN INITIAL PROPERTIES************************
+
+#Email account settings for sending out password resetting emails.
+
+#Email server uses the given username and password to authenticate on TLS connections to host and act as identity.
+#Identity left blank to act as username.
+email_identity = 
+
+email_server = smtp.mydomain.com
+email_server_port = 25
+email_username = sample_admin@mydomain.com
+email_password = abc
+email_from = admin <sample_admin@mydomain.com>
+email_ssl = false
+email_insecure = false
+
+##The initial password of Harbor admin, only works for the first time when Harbor starts. 
+#It has no effect after the first launch of Harbor.
+#Change the admin password from UI after launching Harbor.
+harbor_admin_password = Harbor12345
+
+##By default the auth mode is db_auth, i.e. the credentials are stored in a local database.
+#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server.
+auth_mode = db_auth
+
+#The url for an ldap endpoint.
+ldap_url = ldaps://ldap.mydomain.com
+
+#A user's DN who has the permission to search the LDAP/AD server. 
+#If your LDAP/AD server does not support anonymous search, you should configure this DN and ldap_search_pwd.
+#ldap_searchdn = uid=searchuser,ou=people,dc=mydomain,dc=com
+
+#the password of the ldap_searchdn
+#ldap_search_pwd = password
+
+#The base DN from which to look up a user in LDAP/AD
+ldap_basedn = ou=people,dc=mydomain,dc=com
+
+#Search filter for LDAP/AD, make sure the syntax of the filter is correct.
+#ldap_filter = (objectClass=person)
+
+# The attribute used in a search to match a user, it could be uid, cn, email, sAMAccountName or other attributes depending on your LDAP/AD  
+ldap_uid = uid 
+
+#the scope to search for users, 0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE
+ldap_scope = 2 
+
+#Timeout (in seconds)  when connecting to an LDAP Server. The default value (and most reasonable) is 5 seconds.
+ldap_timeout = 5
+
+#Verify certificate from LDAP server
+ldap_verify_cert = true
+
+#Turn on or off the self-registration feature
+self_registration = on
+
+#The expiration time (in minute) of token created by token service, default is 30 minutes
+token_expiration = 30
+
+#The flag to control what users have permission to create projects
+#The default value "everyone" allows everyone to creates a project. 
+#Set to "adminonly" so that only admin user can create project.
+project_creation_restriction = everyone
+
+#************************END INITIAL PROPERTIES************************
+
+#######Harbor DB configuration section#######
+
+#The address of the Harbor database. Only need to change when using external db.
+db_host = $db_host
+
+#The password for the root user of Harbor DB. Change this before any production use.
+db_password = $db_password
+
+#The port of Harbor database host
+db_port = $db_port
+
+#The user name of Harbor database
+db_user = $db_user
+
+##### End of Harbor DB configuration#######
+
+#The redis server address. Only needed in HA installation.
+redis_url = $redis_url
+
+##########Clair DB configuration############
+
+#Clair DB host address. Only change it when using an exteral DB.
+clair_db_host = $clair_db_host
+
+#The password of the Clair's postgres database. Only effective when Harbor is deployed with Clair.
+#Please update it before deployment. Subsequent update will cause Clair's API server and Harbor unable to access Clair's database.
+clair_db_password = $clair_db_password
+
+#Clair DB connect port
+clair_db_port = $clair_db_port
+
+#Clair DB username
+clair_db_username = $clair_db_username
+
+#Clair default database
+clair_db = $clair_db
+
+##########End of Clair DB configuration############
+
+#The following attributes only need to be set when auth mode is uaa_auth
+uaa_endpoint = $uaa_endpoint
+uaa_clientid = $uaa_clientid
+uaa_clientsecret = $uaa_clientsecret
+uaa_verify_cert = $uaa_verify_cert
+uaa_ca_cert = $uaa_ca_cert
+
+
+### Docker Registry setting ###
+#registry_storage_provider can be: filesystem, s3, gcs, azure, etc.
+registry_storage_provider_name = $registry_storage_provider_name
+#registry_storage_provider_config is a comma separated "key: value" pairs, e.g. "key1: value, key2: value2".
+#Refer to https://docs.docker.com/registry/configuration/#storage for all available configuration.
+registry_storage_provider_config = $registry_storage_provider_config

tools/migration/cfg/utils.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import sys
+import os
+import json
+from string import Template
+
+if sys.version_info[:3][0] == 2:
+    import ConfigParser as ConfigParser
+    import StringIO as StringIO
+
+if sys.version_info[:3][0] == 3:
+    import configparser as ConfigParser
+    import io as StringIO
+
+def read_conf(path):
+    temp_section = "configuration"
+    conf = StringIO.StringIO()
+    conf.write("[%s]\n" % temp_section)
+    conf.write(open(path).read())
+    conf.seek(0, os.SEEK_SET)
+    rcp = ConfigParser.RawConfigParser()
+    rcp.readfp(conf)
+    d = {}
+    for op in rcp.options(temp_section):
+        d[op] = rcp.get(temp_section, op)
+    return d
+
+def get_conf_version(path):
+    d = read_conf(path)
+#    print json.dumps(d,indent=4)
+    if d.has_key("_version"): # >=1.5.0
+        return d["_version"]
+    if not d.has_key("clair_db_password"):
+        return "unsupported"
+    if d.has_key("registry_storage_provider_name"):
+        return "1.4.0"
+    if d.has_key("uaa_endpoint"):
+        return "1.3.0"
+    return "1.2.0"
+
+def render(src, dest, **kw):
+    t = Template(open(src, 'r').read())
+    with open(dest, 'w') as f:
+        f.write(t.substitute(**kw))
+
+    
+    
+    
+
+