From 5bbbb068074dd5f7da644d2a0e7a3326a61dd1b4 Mon Sep 17 00:00:00 2001 From: Yan Date: Sun, 1 Apr 2018 08:53:21 -0700 Subject: [PATCH] add all in one migrator scripts (#4511) --- tools/migration/Dockerfile | 14 ++ tools/migration/cfg/{migrate.py => run.py} | 1 + tools/migration/db/alembic.tpl | 2 +- .../db/migration_cfg/harbor_1_1_0_template | 103 --------- tools/migration/db/migration_cfg/upgrade | 117 ---------- tools/migration/db/run.sh | 60 +++--- tools/migration/docker-entrypoint.sh | 6 + tools/migration/migrator.py | 200 ++++++++++++++++++ 8 files changed, 252 insertions(+), 251 deletions(-) create mode 100644 tools/migration/Dockerfile rename tools/migration/cfg/{migrate.py => run.py} (99%) delete mode 100644 tools/migration/db/migration_cfg/harbor_1_1_0_template delete mode 100755 tools/migration/db/migration_cfg/upgrade create mode 100755 tools/migration/docker-entrypoint.sh create mode 100644 tools/migration/migrator.py diff --git a/tools/migration/Dockerfile b/tools/migration/Dockerfile new file mode 100644 index 000000000..eb225aa3f --- /dev/null +++ b/tools/migration/Dockerfile @@ -0,0 +1,14 @@ +FROM vmware/mariadb-photon:10.2.8 + +RUN tdnf distro-sync -y \ + && tdnf install -y mariadb-devel python2 python2-devel python-pip gcc \ + linux-api-headers glibc-devel binutils zlib-devel openssl-devel \ + && pip install mysqlclient alembic \ + && tdnf clean all \ + && mkdir -p /harbor-migration + +WORKDIR /harbor-migration + +COPY ./ ./ + +ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/tools/migration/cfg/migrate.py b/tools/migration/cfg/run.py similarity index 99% rename from tools/migration/cfg/migrate.py rename to tools/migration/cfg/run.py index 38accd860..b6fdb94ab 100644 --- a/tools/migration/cfg/migrate.py +++ b/tools/migration/cfg/run.py @@ -10,6 +10,7 @@ import utils import importlib import glob import shutil +import sys def main(): target_version = '1.5.0' diff --git a/tools/migration/db/alembic.tpl b/tools/migration/db/alembic.tpl index 6dc1b0515..6a0712def 100644 --- a/tools/migration/db/alembic.tpl +++ b/tools/migration/db/alembic.tpl @@ -3,7 +3,7 @@ echo " [alembic] # path to migration scripts -script_location = migration_harbor +script_location = /harbor-migration/db/migration_harbor # template used to generate migration files # file_template = %%(rev)s_%%(slug)s diff --git a/tools/migration/db/migration_cfg/harbor_1_1_0_template b/tools/migration/db/migration_cfg/harbor_1_1_0_template deleted file mode 100644 index fd246a26a..000000000 --- a/tools/migration/db/migration_cfg/harbor_1_1_0_template +++ /dev/null @@ -1,103 +0,0 @@ -## Configuration file of Harbor - -#The IP address or hostname to access admin UI and registry service. -#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. -hostname = $hostname - -#The protocol for accessing the UI and token/notification service, by default it is http. -#It can be set to https if ssl is enabled on nginx. -ui_url_protocol = $ui_url_protocol - -#The password for the root user of mysql db, change this before any production use. -db_password = $db_password - -#Determine whether the UI should use compressed js files. -#For production, set it to on. For development, set it to off. -use_compressed_js = $use_compressed_js - -#Maximum number of job workers in job service -max_job_workers = $max_job_workers - -#Determine whether or not to generate certificate for the registry's token. -#If the value is on, the prepare script creates new root cert and private key -#for generating token to access the registry. If the value is off the default key/cert will be used. -#This flag also controls the creation of the notary signer's cert. -customize_crt = $customize_crt - -#The path of cert and key files for nginx, they are applied only the protocol is set to https -ssl_cert = $ssl_cert -ssl_cert_key = $ssl_cert_key - -#The path of secretkey storage -secretkey_path = /data - -#Admiral's url, comment this attribute, or set its value to to NA when Harbor is standalone -admiral_url = NA - -#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES -#only take effect in the first boot, the subsequent changes of these properties -#should be performed on web ui -#************************BEGIN INITIAL PROPERTIES************************ - -#Email account settings for sending out password resetting emails. - -#Email server uses the given username and password to authenticate on TLS connections to host and act as identity. -#Identity left blank to act as username. -email_identity = $email_identity - -email_server = $email_server -email_server_port = $email_server_port -email_username = $email_username -email_password = $email_password -email_from = $email_from -email_ssl = $email_ssl - -##The initial password of Harbor admin, only works for the first time when Harbor starts. -#It has no effect after the first launch of Harbor. -#Change the admin password from UI after launching Harbor. -harbor_admin_password = $harbor_admin_password - -##By default the auth mode is db_auth, i.e. the credentials are stored in a local database. -#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server. -auth_mode = $auth_mode - -#The url for an ldap endpoint. -ldap_url = $ldap_url - -#A user's DN who has the permission to search the LDAP/AD server. -#If your LDAP/AD server does not support anonymous search, you should configure this DN and ldap_search_pwd. -#ldap_searchdn = uid=searchuser,ou=people,dc=mydomain,dc=com - -#the password of the ldap_searchdn -#ldap_search_pwd = password - -#The base DN from which to look up a user in LDAP/AD -ldap_basedn = $ldap_basedn - -#Search filter for LDAP/AD, make sure the syntax of the filter is correct. -#ldap_filter = (objectClass=person) - -# The attribute used in a search to match a user, it could be uid, cn, email, sAMAccountName or other attributes depending on your LDAP/AD -ldap_uid = $ldap_uid - -#the scope to search for users, 1-LDAP_SCOPE_BASE, 2-LDAP_SCOPE_ONELEVEL, 3-LDAP_SCOPE_SUBTREE -ldap_scope = $ldap_scope - -#Timeout (in seconds) when connecting to an LDAP Server. The default value (and most reasonable) is 5 seconds. -ldap_timeout = 5 - -#Turn on or off the self-registration feature -self_registration = $self_registration - -#The expiration time (in minute) of token created by token service, default is 30 minutes -token_expiration = $token_expiration - -#The flag to control what users have permission to create projects -#Be default everyone can create a project, set to "adminonly" such that only admin can create project. -project_creation_restriction = $project_creation_restriction - -#Determine whether the job service should verify the ssl cert when it connects to a remote registry. -#Set this flag to off when the remote registry uses a self-signed or untrusted certificate. -verify_remote_cert = $verify_remote_cert -#************************BEGIN INITIAL PROPERTIES************************ -############# diff --git a/tools/migration/db/migration_cfg/upgrade b/tools/migration/db/migration_cfg/upgrade deleted file mode 100755 index 15675d07e..000000000 --- a/tools/migration/db/migration_cfg/upgrade +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -from string import Template -import string -import os -import sys -import argparse -from io import open - -if sys.version_info[:3][0] == 2: - import ConfigParser as ConfigParser - import StringIO as StringIO - -if sys.version_info[:3][0] == 3: - import configparser as ConfigParser - import io as StringIO - -parser = argparse.ArgumentParser() -parser.add_argument('--source-loc', dest='source_loc', type=str,help="the path of Harbor 0.5.0 configuration file") -parser.add_argument('--source-version', dest='source_ver', type=str,help="the Harbor instance is to be deployed with notary") -parser.add_argument('--target-loc', dest='target_loc', type=str,help="the path of Harbor 1.1.x configuration file") -parser.add_argument('--target-version', dest='target_ver', type=str, help="the Harbor instance is to be deployed with notary") -upgrade_args = parser.parse_args() - -# NOTE: the script only support to upgrade from 0.5.0.to 1.1.x. -def validate(): - if upgrade_args.source_ver == '0.5.0' and upgrade_args.target_ver == '1.1.x': - return - raise Exception("Unable to support upgrade from %s to %s" % (upgrade_args.source_ver, upgrade_args.target_ver)) - -validate() - -conf = StringIO.StringIO() -conf.write("[configuration]\n") -conf.write(open(upgrade_args.source_loc).read()) -conf.seek(0, os.SEEK_SET) -rcp = ConfigParser.RawConfigParser() -rcp.readfp(conf) - -hostname = rcp.get("configuration", "hostname") -ui_url_protocol = rcp.get("configuration", "ui_url_protocol") -email_identity = rcp.get("configuration", "email_identity") -email_server = rcp.get("configuration", "email_server") -email_server_port = rcp.get("configuration", "email_server_port") -email_username = rcp.get("configuration", "email_username") -email_password = rcp.get("configuration", "email_password") -email_from = rcp.get("configuration", "email_from") -email_ssl = rcp.get("configuration", "email_ssl") -harbor_admin_password = rcp.get("configuration", "harbor_admin_password") -auth_mode = rcp.get("configuration", "auth_mode") -ldap_url = rcp.get("configuration", "ldap_url") -ldap_basedn = rcp.get("configuration", "ldap_basedn") -ldap_uid = rcp.get("configuration", "ldap_uid") -ldap_scope = rcp.get("configuration", "ldap_scope") -db_password = rcp.get("configuration", "db_password") -self_registration = rcp.get("configuration", "self_registration") -use_compressed_js = rcp.get("configuration", "use_compressed_js") -max_job_workers = rcp.get("configuration", "max_job_workers") -token_expiration = rcp.get("configuration", "token_expiration") -verify_remote_cert = rcp.get("configuration", "verify_remote_cert") -customize_crt = rcp.get("configuration", "customize_crt") -project_creation_restriction = rcp.get("configuration", "project_creation_restriction") -ssl_cert = rcp.get("configuration", "ssl_cert") -ssl_cert_key = rcp.get("configuration", "ssl_cert_key") - -def delfile(src): - if os.path.isfile(src): - try: - os.remove(src) - print("Clearing the configuration file: %s" % src) - except: - pass - elif os.path.isdir(src): - for item in os.listdir(src): - itemsrc=os.path.join(src,item) - delfile(itemsrc) - -def render(src, dest, **kw): - t = Template(open(src, 'r').read()) - with open(dest, 'w') as f: - f.write(t.substitute(**kw)) - print("Generated configuration file: %s" % dest) - -delfile(upgrade_args.target_loc) - -base_dir = os.path.dirname(__file__) -config_template = os.path.join(base_dir, "harbor_1_1_0_template") - -render(config_template, - upgrade_args.target_loc, - hostname=hostname, - ui_url_protocol=ui_url_protocol, - db_password=db_password, - use_compressed_js=use_compressed_js, - max_job_workers=max_job_workers, - customize_crt=customize_crt, - ssl_cert=ssl_cert, - ssl_cert_key=ssl_cert_key, - admiral_url='', - email_identity=email_identity, - email_server=email_server, - email_server_port=email_server_port, - email_username=email_username, - email_password=email_password, - email_from=email_from, - email_ssl=email_ssl, - harbor_admin_password=harbor_admin_password, - auth_mode=auth_mode, - ldap_url=ldap_url, - ldap_basedn=ldap_basedn, - ldap_uid=ldap_uid, - ldap_scope=ldap_scope, - self_registration=self_registration, - token_expiration=token_expiration, - project_creation_restriction=project_creation_restriction, - verify_remote_cert=verify_remote_cert - ) diff --git a/tools/migration/db/run.sh b/tools/migration/db/run.sh index 1586ed5ee..52d370d1e 100755 --- a/tools/migration/db/run.sh +++ b/tools/migration/db/run.sh @@ -1,12 +1,12 @@ #!/bin/bash -export PYTHONPATH=$PYTHONPATH:/harbor-migration +export PYTHONPATH=$PYTHONPATH:/harbor-migration/db if [ -z "$DB_USR" -o -z "$DB_PWD" ]; then echo "DB_USR or DB_PWD not set, exiting..." exit 1 fi -source ./alembic.tpl > ./alembic.ini +source /harbor-migration/db/alembic.tpl > /harbor-migration/db/alembic.ini DBCNF="-hlocalhost -u${DB_USR}" @@ -23,42 +23,41 @@ if [[ $1 = "help" || $1 = "h" || $# = 0 ]]; then exit 0 fi -if [[ ( $1 = "up" || $1 = "upgrade" ) && ${SKIP_CONFIRM} != "y" ]]; then - echo "Please backup before upgrade." - read -p "Enter y to continue updating or n to abort:" ans - case $ans in - [Yy]* ) - ;; - [Nn]* ) - exit 0 - ;; - * ) echo "illegal answer: $ans. Upgrade abort!!" - exit 1 - ;; - esac - -fi +# if [[ ( $1 = "up" || $1 = "upgrade" ) && ${SKIP_CONFIRM} != "y" ]]; then +# echo "Please backup before upgrade." +# read -p "Enter y to continue updating or n to abort:" ans +# case $ans in +# [Yy]* ) +# ;; +# [Nn]* ) +# exit 0 +# ;; +# * ) echo "illegal answer: $ans. Upgrade abort!!" +# exit 1 +# ;; +# esac +# fi echo 'Trying to start mysql server...' chown -R 10000:10000 /var/lib/mysql mysqld & +echo 'Waiting for MySQL start...' for i in {60..0}; do mysqladmin -u$DB_USR -p$DB_PWD processlist >/dev/null 2>&1 if [ $? = 0 ]; then break fi - echo 'Waiting for MySQL start...' sleep 1 done if [ "$i" = 0 ]; then echo "timeout. Can't run mysql server." if [[ $1 = "test" ]]; then - echo "test failed." + echo "DB test failed." fi exit 1 fi if [[ $1 = "test" ]]; then - echo "test passed." + echo "DB test passed." exit 0 fi @@ -84,39 +83,40 @@ up|upgrade) mysql $DBCNF -e "insert into registry.alembic_version values ('0.1.1')" fi fi - alembic -c ./alembic.ini current - alembic -c ./alembic.ini upgrade ${VERSION} + alembic -c /harbor-migration/db/alembic.ini current + alembic -c /harbor-migration/db/alembic.ini upgrade ${VERSION} rc="$?" - alembic -c ./alembic.ini current + alembic -c /harbor-migration/db/alembic.ini current echo "Upgrade performed." - echo $rc exit $rc ;; backup) echo "Performing backup..." - mysqldump $DBCNF --add-drop-database --databases registry > ./backup/registry.sql + mysqldump $DBCNF --add-drop-database --databases registry > /harbor-migration/backup/registry.sql + rc="$?" echo "Backup performed." + exit $rc ;; export) echo "Performing export..." - ./export --dbuser ${DB_USR} --dbpwd ${DB_PWD} --exportpath ${EXPORTPATH} + /harbor-migration/db/export --dbuser ${DB_USR} --dbpwd ${DB_PWD} --exportpath ${EXPORTPATH} rc="$?" echo "Export performed." - echo $rc exit $rc ;; mapprojects) echo "Performing map projects..." - ./mapprojects --dbuser ${DB_USR} --dbpwd ${DB_PWD} --mapprojectsfile ${MAPPROJECTFILE} + /harbor-migration/db/mapprojects --dbuser ${DB_USR} --dbpwd ${DB_PWD} --mapprojectsfile ${MAPPROJECTFILE} rc="$?" echo "Map projects performed." - echo $rc exit $rc ;; restore) echo "Performing restore..." - mysql $DBCNF < ./backup/registry.sql + mysql $DBCNF < /harbor-migration/db/backup/registry.sql + rc="$?" echo "Restore performed." + exit $rc ;; *) echo "unknown option" diff --git a/tools/migration/docker-entrypoint.sh b/tools/migration/docker-entrypoint.sh new file mode 100755 index 000000000..956acc33d --- /dev/null +++ b/tools/migration/docker-entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +python ./migrator.py "$@" + +exec "$@" \ No newline at end of file diff --git a/tools/migration/migrator.py b/tools/migration/migrator.py new file mode 100644 index 000000000..f175be79b --- /dev/null +++ b/tools/migration/migrator.py @@ -0,0 +1,200 @@ +import abc +import subprocess +from optparse import OptionParser +from shutil import copyfile +import os +import sys +import argparse + +RC_VALIDATE = 101 +RC_UP = 102 +RC_DOWN = 103 +RC_BACKUP = 104 +RC_RESTORE = 105 +RC_UNNKNOW_TYPE = 106 +RC_GEN = 110 + +class DBMigrator(): + + def __init__(self, target): + self.target = target + self.script = "./db/run.sh" + + def backup(self): + return run_cmd(self.script + " backup") == 0 + + def restore(self): + return run_cmd(self.script + " restore") == 0 + + def up(self): + cmd = self.script + " up" + if self.target != '': + cmd = cmd + " " + self.target + return run_cmd(cmd) == 0 + + def validate(self): + return run_cmd(self.script + " test") == 0 + +class CfgMigrator(): + + def __init__(self, target): + self.target = target + self.cfg_path = "/harbor-migration/harbor-cfg/harbor.cfg" + self.backup_path = "/harbor-migration/backup" + + def backup(self): + try: + copyfile(self.cfg_path, self.backup_path+"/harbor.cfg") + print ("Success to backup harbor.cfg.") + return True + except Exception, e: + print ("Back up error: %s" % str(e)) + return False + + def restore(self): + if not os.path.exists(self.backup_path+"/harbor.cfg"): + print ("Unable to restore as there is no harbor.cfg") + return False + try: + copyfile(self.backup_path+"/harbor.cfg", self.cfg_path) + print ("Success to restore harbor.cfg.") + return True + except Exception, e: + print ("Restore error: %s" % str(e)) + return False + + def up(self): + if not os.path.exists(self.cfg_path): + print ("Skip cfg up as no harbor.cfg in the path.") + return True + cmd = "python ./cfg/run.py --input " + self.cfg_path + if self.target != '': + cmd = cmd + " --target " + self.target + return run_cmd(cmd) == 0 + + def validate(self): + if not os.path.exists(self.cfg_path): + print ("Unable to loacte the harbor.cfg, please check.") + return False + print ("Success to validate harbor.cfg.") + return True + +class Parameters(object): + def __init__(self): + self.db_user = os.getenv('DB_USR', '') + self.db_pwd = os.getenv('DB_PWD', '') + self.skip_confirm = os.getenv('SKIP_CONFIRM', 'n') + self.is_migrate_db = True + self.is_migrate_cfg = True + self.target_version = '' + self.action = '' + self.init_from_input() + + def is_action(self, action): + if action == "test" or action == "backup" or action == "restore" or action == "up": + return True + else: + return False + + def parse_input(self): + argv_len = len(sys.argv[1:]) + last_argv = sys.argv[argv_len:][0] + if not self.is_action(last_argv): + print ("Fail to parse input: the last parameter should in test:up:restore:backup") + sys.exit(RC_GEN) + + if last_argv == 'up': + if self.skip_confirm != 'y': + if not pass_skip_confirm(): + sys.exit(RC_GEN) + + if argv_len == 1: + return (True, True, '', last_argv) + + parser = argparse.ArgumentParser(description='migrator of harbor') + parser.add_argument('--db', action="store_true", dest='is_migrate_db', required=False, default=False, help='The flag to upgrade db.') + parser.add_argument('--cfg', action="store_true", dest='is_migrate_cfg', required=False, default=False, help='The flag to upgrede cfg.') + parser.add_argument('--version', action="store", dest='target_version', required=False, default='', help='The target version that the harbor will be migrated to.') + + args = parser.parse_args(sys.argv[1:argv_len]) + args.action = last_argv + return (args.is_migrate_db, args.is_migrate_cfg, args.target_version, args.action) + + def init_from_input(self): + (self.is_migrate_db, self.is_migrate_cfg, self.target_version, self.action) = self.parse_input() + +def run_cmd(cmd): + return os.system(cmd) + +def pass_skip_confirm(): + valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} + message = "Please backup before upgrade, \nEnter y to continue updating or n to abort: " + while True: + sys.stdout.write(message) + choice = raw_input().lower() + if choice == '': + return False + elif choice in valid: + return valid[choice] + else: + sys.stdout.write("Please respond with 'yes' or 'no' " + "(or 'y' or 'n').\n") + +def main(): + commandline_input = Parameters() + + db_migrator = DBMigrator(commandline_input.target_version) + cfg_migrator = CfgMigrator(commandline_input.target_version) + + try: + # test + if commandline_input.action == "test": + if commandline_input.is_migrate_db: + if not db_migrator.validate(): + print ("Fail to validate: please make sure your DB auth is correct.") + sys.exit(RC_VALIDATE) + + if commandline_input.is_migrate_cfg: + if not cfg_migrator.validate(): + print ("Fail to validate: please make sure your cfg path is correct.") + sys.exit(RC_VALIDATE) + + # backup + elif commandline_input.action == "backup": + if commandline_input.is_migrate_db: + if not db_migrator.backup(): + sys.exit(RC_BACKUP) + + if commandline_input.is_migrate_cfg: + if not cfg_migrator.backup(): + sys.exit(RC_BACKUP) + + # up + elif commandline_input.action == "up": + if commandline_input.is_migrate_db: + if not db_migrator.up(): + sys.exit(RC_UP) + + if commandline_input.is_migrate_cfg: + if not cfg_migrator.up(): + sys.exit(RC_UP) + + # restore + elif commandline_input.action == "restore": + if commandline_input.is_migrate_db: + if not db_migrator.restore(): + sys.exit(RC_RESTORE) + + if commandline_input.is_migrate_cfg: + if not cfg_migrator.restore(): + sys.exit(RC_RESTORE) + + else: + print ("Unknow action type: " + str(commandline_input.action)) + sys.exit(RC_UNNKNOW_TYPE) + except Exception as ex: + print ("Migrator fail to execute, err: " + ex.message) + sys.exit(RC_GEN) + +if __name__ == '__main__': + main()