add all in one migrator scripts (#4511)

tools/migration/Dockerfile

@@ -0,0 +1,14 @@
+FROM vmware/mariadb-photon:10.2.8
+
+RUN tdnf distro-sync -y \
+    && tdnf install -y mariadb-devel python2 python2-devel python-pip gcc \
+    linux-api-headers glibc-devel binutils zlib-devel openssl-devel \
+    && pip install mysqlclient alembic \
+    && tdnf clean all \
+    && mkdir -p /harbor-migration
+
+WORKDIR /harbor-migration
+
+COPY ./ ./
+
+ENTRYPOINT ["./docker-entrypoint.sh"] 

tools/migration/cfg/run.py

@@ -10,6 +10,7 @@ import utils
 import importlib
 import glob
 import shutil
+import sys
 
 def main():
     target_version = '1.5.0'

tools/migration/db/alembic.tpl

@@ -3,7 +3,7 @@ echo "
 
 [alembic]
 # path to migration scripts
-script_location = migration_harbor
+script_location = /harbor-migration/db/migration_harbor
 
 # template used to generate migration files
 # file_template = %%(rev)s_%%(slug)s

tools/migration/db/migration_cfg/harbor_1_1_0_template

@@ -1,103 +0,0 @@
-## Configuration file of Harbor
-
-#The IP address or hostname to access admin UI and registry service.
-#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
-hostname = $hostname
-
-#The protocol for accessing the UI and token/notification service, by default it is http.
-#It can be set to https if ssl is enabled on nginx.
-ui_url_protocol = $ui_url_protocol
-
-#The password for the root user of mysql db, change this before any production use.
-db_password = $db_password
-
-#Determine whether the UI should use compressed js files.
-#For production, set it to on. For development, set it to off.
-use_compressed_js = $use_compressed_js
-
-#Maximum number of job workers in job service
-max_job_workers = $max_job_workers
-
-#Determine whether or not to generate certificate for the registry's token.
-#If the value is on, the prepare script creates new root cert and private key
-#for generating token to access the registry. If the value is off the default key/cert will be used.
-#This flag also controls the creation of the notary signer's cert.
-customize_crt = $customize_crt
-
-#The path of cert and key files for nginx, they are applied only the protocol is set to https
-ssl_cert = $ssl_cert
-ssl_cert_key = $ssl_cert_key
-
-#The path of secretkey storage
-secretkey_path = /data
-
-#Admiral's url, comment this attribute, or set its value to to NA when Harbor is standalone
-admiral_url = NA
-
-#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES
-#only take effect in the first boot, the subsequent changes of these properties
-#should be performed on web ui
-#************************BEGIN INITIAL PROPERTIES************************
-
-#Email account settings for sending out password resetting emails.
-
-#Email server uses the given username and password to authenticate on TLS connections to host and act as identity.
-#Identity left blank to act as username.
-email_identity = $email_identity
-
-email_server = $email_server
-email_server_port = $email_server_port
-email_username = $email_username
-email_password = $email_password
-email_from = $email_from
-email_ssl = $email_ssl
-
-##The initial password of Harbor admin, only works for the first time when Harbor starts.
-#It has no effect after the first launch of Harbor.
-#Change the admin password from UI after launching Harbor.
-harbor_admin_password = $harbor_admin_password
-
-##By default the auth mode is db_auth, i.e. the credentials are stored in a local database.
-#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server.
-auth_mode = $auth_mode
-
-#The url for an ldap endpoint.
-ldap_url = $ldap_url
-
-#A user's DN who has the permission to search the LDAP/AD server.
-#If your LDAP/AD server does not support anonymous search, you should configure this DN and ldap_search_pwd.
-#ldap_searchdn = uid=searchuser,ou=people,dc=mydomain,dc=com
-
-#the password of the ldap_searchdn
-#ldap_search_pwd = password
-
-#The base DN from which to look up a user in LDAP/AD
-ldap_basedn = $ldap_basedn
-
-#Search filter for LDAP/AD, make sure the syntax of the filter is correct.
-#ldap_filter = (objectClass=person)
-
-# The attribute used in a search to match a user, it could be uid, cn, email, sAMAccountName or other attributes depending on your LDAP/AD
-ldap_uid = $ldap_uid
-
-#the scope to search for users, 1-LDAP_SCOPE_BASE, 2-LDAP_SCOPE_ONELEVEL, 3-LDAP_SCOPE_SUBTREE
-ldap_scope = $ldap_scope
-
-#Timeout (in seconds)  when connecting to an LDAP Server. The default value (and most reasonable) is 5 seconds.
-ldap_timeout = 5
-
-#Turn on or off the self-registration feature
-self_registration = $self_registration
-
-#The expiration time (in minute) of token created by token service, default is 30 minutes
-token_expiration = $token_expiration
-
-#The flag to control what users have permission to create projects
-#Be default everyone can create a project, set to "adminonly" such that only admin can create project.
-project_creation_restriction = $project_creation_restriction
-
-#Determine whether the job service should verify the ssl cert when it connects to a remote registry.
-#Set this flag to off when the remote registry uses a self-signed or untrusted certificate.
-verify_remote_cert = $verify_remote_cert
-#************************BEGIN INITIAL PROPERTIES************************
-#############

tools/migration/db/migration_cfg/upgrade

@@ -1,117 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-from string import Template
-import string
-import os
-import sys
-import argparse
-from io import open
-
-if sys.version_info[:3][0] == 2:
-    import ConfigParser as ConfigParser
-    import StringIO as StringIO
-
-if sys.version_info[:3][0] == 3:
-    import configparser as ConfigParser
-    import io as StringIO
-
-parser = argparse.ArgumentParser()
-parser.add_argument('--source-loc', dest='source_loc', type=str,help="the path of Harbor 0.5.0 configuration file")
-parser.add_argument('--source-version', dest='source_ver', type=str,help="the Harbor instance is to be deployed with notary")
-parser.add_argument('--target-loc', dest='target_loc', type=str,help="the path of Harbor 1.1.x configuration file")
-parser.add_argument('--target-version', dest='target_ver', type=str, help="the Harbor instance is to be deployed with notary")
-upgrade_args = parser.parse_args()
-
-# NOTE: the script only support to upgrade from 0.5.0.to 1.1.x.
-def validate():
-    if upgrade_args.source_ver == '0.5.0' and upgrade_args.target_ver == '1.1.x':
-        return
-    raise Exception("Unable to support upgrade from %s to %s" % (upgrade_args.source_ver, upgrade_args.target_ver))
-
-validate()
-
-conf = StringIO.StringIO()
-conf.write("[configuration]\n")
-conf.write(open(upgrade_args.source_loc).read())
-conf.seek(0, os.SEEK_SET)
-rcp = ConfigParser.RawConfigParser()
-rcp.readfp(conf)
-
-hostname = rcp.get("configuration", "hostname")
-ui_url_protocol = rcp.get("configuration", "ui_url_protocol")
-email_identity = rcp.get("configuration", "email_identity")
-email_server = rcp.get("configuration", "email_server")
-email_server_port = rcp.get("configuration", "email_server_port")
-email_username = rcp.get("configuration", "email_username")
-email_password = rcp.get("configuration", "email_password")
-email_from = rcp.get("configuration", "email_from")
-email_ssl = rcp.get("configuration", "email_ssl")
-harbor_admin_password = rcp.get("configuration", "harbor_admin_password")
-auth_mode = rcp.get("configuration", "auth_mode")
-ldap_url = rcp.get("configuration", "ldap_url")
-ldap_basedn = rcp.get("configuration", "ldap_basedn")
-ldap_uid = rcp.get("configuration", "ldap_uid")
-ldap_scope = rcp.get("configuration", "ldap_scope")
-db_password = rcp.get("configuration", "db_password")
-self_registration = rcp.get("configuration", "self_registration")
-use_compressed_js = rcp.get("configuration", "use_compressed_js")
-max_job_workers = rcp.get("configuration", "max_job_workers")
-token_expiration = rcp.get("configuration", "token_expiration")
-verify_remote_cert = rcp.get("configuration", "verify_remote_cert")
-customize_crt = rcp.get("configuration", "customize_crt")
-project_creation_restriction = rcp.get("configuration", "project_creation_restriction")
-ssl_cert = rcp.get("configuration", "ssl_cert")
-ssl_cert_key = rcp.get("configuration", "ssl_cert_key")
-
-def delfile(src):
-    if os.path.isfile(src):
-        try:
-            os.remove(src)
-            print("Clearing the configuration file: %s" % src)
-        except:
-            pass
-    elif os.path.isdir(src):
-        for item in os.listdir(src):
-            itemsrc=os.path.join(src,item)
-            delfile(itemsrc)
-
-def render(src, dest, **kw):
-    t = Template(open(src, 'r').read())
-    with open(dest, 'w') as f:
-        f.write(t.substitute(**kw))
-    print("Generated configuration file: %s" % dest)
-
-delfile(upgrade_args.target_loc)
-
-base_dir = os.path.dirname(__file__)
-config_template = os.path.join(base_dir, "harbor_1_1_0_template")
-
-render(config_template,
-        upgrade_args.target_loc,
-        hostname=hostname,
-        ui_url_protocol=ui_url_protocol,
-        db_password=db_password,
-        use_compressed_js=use_compressed_js,
-        max_job_workers=max_job_workers,
-        customize_crt=customize_crt,
-        ssl_cert=ssl_cert,
-        ssl_cert_key=ssl_cert_key,
-        admiral_url='',
-        email_identity=email_identity,
-        email_server=email_server,
-        email_server_port=email_server_port,
-        email_username=email_username,
-        email_password=email_password,
-        email_from=email_from,
-        email_ssl=email_ssl,
-        harbor_admin_password=harbor_admin_password,
-        auth_mode=auth_mode,
-        ldap_url=ldap_url,
-        ldap_basedn=ldap_basedn,
-        ldap_uid=ldap_uid,
-        ldap_scope=ldap_scope,
-        self_registration=self_registration,
-        token_expiration=token_expiration,
-        project_creation_restriction=project_creation_restriction,
-        verify_remote_cert=verify_remote_cert
-    )

tools/migration/db/run.sh

@@ -1,12 +1,12 @@
 #!/bin/bash
 
-export PYTHONPATH=$PYTHONPATH:/harbor-migration
+export PYTHONPATH=$PYTHONPATH:/harbor-migration/db
 if [ -z "$DB_USR" -o -z "$DB_PWD" ]; then
     echo "DB_USR or DB_PWD not set, exiting..."
     exit 1
 fi
 
-source ./alembic.tpl > ./alembic.ini
+source /harbor-migration/db/alembic.tpl > /harbor-migration/db/alembic.ini
 
 DBCNF="-hlocalhost -u${DB_USR}"
 
@@ -23,42 +23,41 @@ if [[ $1 = "help" || $1 = "h" || $# = 0 ]]; then
     exit 0
 fi
 
-if [[ ( $1 = "up" || $1 = "upgrade" ) && ${SKIP_CONFIRM} != "y" ]]; then
+# if [[ ( $1 = "up" || $1 = "upgrade" ) && ${SKIP_CONFIRM} != "y" ]]; then
-    echo "Please backup before upgrade."
+#     echo "Please backup before upgrade."
-    read -p "Enter y to continue updating or n to abort:" ans
+#     read -p "Enter y to continue updating or n to abort:" ans
-    case $ans in
+#     case $ans in
-        [Yy]* )
+#         [Yy]* )
-            ;;
+#             ;;
-        [Nn]* )
+#         [Nn]* )
-            exit 0
+#             exit 0
-            ;;
+#             ;;
-        * ) echo "illegal answer: $ans. Upgrade abort!!"
+#         * ) echo "illegal answer: $ans. Upgrade abort!!"
-            exit 1
+#             exit 1
-            ;;
+#             ;;
-    esac
+#     esac
-
+# fi
-fi
 
 echo 'Trying to start mysql server...'
 chown -R 10000:10000 /var/lib/mysql
 mysqld &
+echo 'Waiting for MySQL start...'
 for i in {60..0}; do
     mysqladmin -u$DB_USR -p$DB_PWD processlist >/dev/null 2>&1
     if [ $? = 0 ]; then
         break
     fi
-    echo 'Waiting for MySQL start...'
     sleep 1
 done
 if [ "$i" = 0 ]; then
     echo "timeout. Can't run mysql server."
     if [[ $1 = "test" ]]; then
-        echo "test failed."
+        echo "DB test failed."
     fi
     exit 1
 fi
 if [[ $1 = "test" ]]; then
-    echo "test passed."
+    echo "DB test passed."
     exit 0
 fi
 
@@ -84,39 +83,40 @@ up|upgrade)
             mysql $DBCNF -e "insert into registry.alembic_version values ('0.1.1')"
         fi
     fi
-    alembic -c ./alembic.ini current
+    alembic -c /harbor-migration/db/alembic.ini current
-    alembic -c ./alembic.ini upgrade ${VERSION}
+    alembic -c /harbor-migration/db/alembic.ini upgrade ${VERSION}
     rc="$?"
-    alembic -c ./alembic.ini current	
+    alembic -c /harbor-migration/db/alembic.ini current	
     echo "Upgrade performed."
-    echo $rc
     exit $rc	
     ;;
 backup)
     echo "Performing backup..."
-    mysqldump $DBCNF --add-drop-database --databases registry > ./backup/registry.sql
+    mysqldump $DBCNF --add-drop-database --databases registry > /harbor-migration/backup/registry.sql
+    rc="$?"
     echo "Backup performed."
+    exit $rc
     ;;
 export)
     echo "Performing export..."
-    ./export --dbuser ${DB_USR} --dbpwd ${DB_PWD} --exportpath ${EXPORTPATH}
+    /harbor-migration/db/export --dbuser ${DB_USR} --dbpwd ${DB_PWD} --exportpath ${EXPORTPATH}
     rc="$?"
     echo "Export performed."
-    echo $rc
     exit $rc
     ;;
 mapprojects)
     echo "Performing map projects..."
-    ./mapprojects --dbuser ${DB_USR} --dbpwd ${DB_PWD} --mapprojectsfile ${MAPPROJECTFILE}
+    /harbor-migration/db/mapprojects --dbuser ${DB_USR} --dbpwd ${DB_PWD} --mapprojectsfile ${MAPPROJECTFILE}
     rc="$?"
     echo "Map projects performed."
-    echo $rc
     exit $rc
     ;;
 restore)
     echo "Performing restore..."
-    mysql $DBCNF < ./backup/registry.sql
+    mysql $DBCNF < /harbor-migration/db/backup/registry.sql
+    rc="$?"
     echo "Restore performed."
+    exit $rc
     ;;
 *)
     echo "unknown option"

tools/migration/docker-entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+python ./migrator.py "$@"
+
+exec "$@"

tools/migration/migrator.py

@@ -0,0 +1,200 @@
+import abc
+import subprocess
+from optparse import OptionParser
+from shutil import copyfile
+import os
+import sys
+import argparse
+
+RC_VALIDATE = 101
+RC_UP = 102
+RC_DOWN = 103
+RC_BACKUP = 104
+RC_RESTORE = 105
+RC_UNNKNOW_TYPE = 106
+RC_GEN = 110
+
+class DBMigrator():
+
+    def __init__(self, target):
+        self.target = target
+        self.script = "./db/run.sh"
+
+    def backup(self):
+        return run_cmd(self.script + " backup") == 0
+
+    def restore(self):
+        return run_cmd(self.script + " restore") == 0
+
+    def up(self):
+        cmd = self.script + " up"
+        if self.target != '':
+            cmd = cmd + " " + self.target
+        return run_cmd(cmd) == 0
+
+    def validate(self):
+        return run_cmd(self.script + " test") == 0
+
+class CfgMigrator():
+
+    def __init__(self, target):
+        self.target = target
+        self.cfg_path = "/harbor-migration/harbor-cfg/harbor.cfg"
+        self.backup_path = "/harbor-migration/backup"
+
+    def backup(self):
+        try:
+            copyfile(self.cfg_path, self.backup_path+"/harbor.cfg")
+            print ("Success to backup harbor.cfg.")
+            return True
+        except Exception, e:
+            print ("Back up error: %s" % str(e))
+            return False 
+
+    def restore(self):
+        if not os.path.exists(self.backup_path+"/harbor.cfg"):
+            print ("Unable to restore as there is no harbor.cfg")
+            return False
+        try:
+            copyfile(self.backup_path+"/harbor.cfg", self.cfg_path)
+            print ("Success to restore harbor.cfg.")
+            return True
+        except Exception, e:
+            print ("Restore error: %s" % str(e))
+            return False
+
+    def up(self):
+        if not os.path.exists(self.cfg_path):
+            print ("Skip cfg up as no harbor.cfg in the path.")
+            return True
+        cmd = "python ./cfg/run.py --input " + self.cfg_path
+        if self.target != '':
+            cmd = cmd + " --target " + self.target
+        return run_cmd(cmd) == 0
+
+    def validate(self):
+        if not os.path.exists(self.cfg_path):
+            print ("Unable to loacte the harbor.cfg, please check.")
+            return False
+        print ("Success to validate harbor.cfg.")
+        return True
+
+class Parameters(object):
+    def __init__(self):    
+        self.db_user = os.getenv('DB_USR', '')
+        self.db_pwd = os.getenv('DB_PWD', '')
+        self.skip_confirm = os.getenv('SKIP_CONFIRM', 'n')
+        self.is_migrate_db = True
+        self.is_migrate_cfg = True
+        self.target_version = ''
+        self.action = ''
+        self.init_from_input()
+
+    def is_action(self, action):
+        if action == "test" or action == "backup" or action == "restore" or action == "up":
+            return True     
+        else:
+            return False   
+
+    def parse_input(self):
+        argv_len = len(sys.argv[1:])
+        last_argv = sys.argv[argv_len:][0]
+        if not self.is_action(last_argv):
+            print ("Fail to parse input: the last parameter should in test:up:restore:backup")
+            sys.exit(RC_GEN) 
+
+        if last_argv == 'up':
+            if self.skip_confirm != 'y':
+                if not pass_skip_confirm():
+                    sys.exit(RC_GEN) 
+        
+        if argv_len == 1:
+            return (True, True, '', last_argv)
+
+        parser = argparse.ArgumentParser(description='migrator of harbor') 
+        parser.add_argument('--db', action="store_true", dest='is_migrate_db', required=False, default=False, help='The flag to upgrade db.')
+        parser.add_argument('--cfg', action="store_true", dest='is_migrate_cfg', required=False, default=False, help='The flag to upgrede cfg.')
+        parser.add_argument('--version', action="store", dest='target_version', required=False, default='', help='The target version that the harbor will be migrated to.')         
+
+        args = parser.parse_args(sys.argv[1:argv_len])
+        args.action = last_argv
+        return (args.is_migrate_db, args.is_migrate_cfg, args.target_version, args.action)
+
+    def init_from_input(self):
+        (self.is_migrate_db, self.is_migrate_cfg, self.target_version, self.action) = self.parse_input()
+
+def run_cmd(cmd):
+    return os.system(cmd)
+
+def pass_skip_confirm():
+    valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
+    message = "Please backup before upgrade, \nEnter y to continue updating or n to abort: "
+    while True:
+        sys.stdout.write(message)
+        choice = raw_input().lower()
+        if choice == '':
+            return False
+        elif choice in valid:
+            return valid[choice]
+        else:
+            sys.stdout.write("Please respond with 'yes' or 'no' "
+                             "(or 'y' or 'n').\n")
+
+def main():
+    commandline_input = Parameters()
+
+    db_migrator = DBMigrator(commandline_input.target_version)
+    cfg_migrator = CfgMigrator(commandline_input.target_version)
+
+    try:
+        # test
+        if commandline_input.action == "test":
+            if commandline_input.is_migrate_db:
+                if not db_migrator.validate():
+                    print ("Fail to validate: please make sure your DB auth is correct.")
+                    sys.exit(RC_VALIDATE)                    
+
+            if commandline_input.is_migrate_cfg:
+                if not cfg_migrator.validate():                 
+                    print ("Fail to validate: please make sure your cfg path is correct.")
+                    sys.exit(RC_VALIDATE) 
+        
+        # backup
+        elif commandline_input.action == "backup":
+            if commandline_input.is_migrate_db:
+                if not db_migrator.backup():
+                    sys.exit(RC_BACKUP)                    
+
+            if commandline_input.is_migrate_cfg:
+                if not cfg_migrator.backup():                 
+                    sys.exit(RC_BACKUP)         
+        
+        # up
+        elif commandline_input.action == "up":
+            if commandline_input.is_migrate_db:
+                if not db_migrator.up():
+                    sys.exit(RC_UP)                    
+
+            if commandline_input.is_migrate_cfg:
+                if not cfg_migrator.up():                 
+                    sys.exit(RC_UP)
+        
+        # restore
+        elif commandline_input.action == "restore":
+            if commandline_input.is_migrate_db:
+                if not db_migrator.restore():
+                    sys.exit(RC_RESTORE)                    
+
+            if commandline_input.is_migrate_cfg:
+                if not cfg_migrator.restore():                 
+                    sys.exit(RC_RESTORE)
+        
+        else:
+            print ("Unknow action type: " + str(commandline_input.action))
+            sys.exit(RC_UNNKNOW_TYPE)     
+    except Exception as ex:
+        print ("Migrator fail to execute, err: " + ex.message)
+        sys.exit(RC_GEN)
+
+if __name__ == '__main__':
+    main()