add all in one migrator scripts (#4511)

This commit is contained in:
Yan 2018-04-01 08:53:21 -07:00 committed by GitHub
parent 199ebdb751
commit 5bbbb06807
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 252 additions and 251 deletions

View File

@ -0,0 +1,14 @@
FROM vmware/mariadb-photon:10.2.8
RUN tdnf distro-sync -y \
&& tdnf install -y mariadb-devel python2 python2-devel python-pip gcc \
linux-api-headers glibc-devel binutils zlib-devel openssl-devel \
&& pip install mysqlclient alembic \
&& tdnf clean all \
&& mkdir -p /harbor-migration
WORKDIR /harbor-migration
COPY ./ ./
ENTRYPOINT ["./docker-entrypoint.sh"]

View File

@ -10,6 +10,7 @@ import utils
import importlib
import glob
import shutil
import sys
def main():
target_version = '1.5.0'

View File

@ -3,7 +3,7 @@ echo "
[alembic]
# path to migration scripts
script_location = migration_harbor
script_location = /harbor-migration/db/migration_harbor
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s

View File

@ -1,103 +0,0 @@
## Configuration file of Harbor
#The IP address or hostname to access admin UI and registry service.
#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname = $hostname
#The protocol for accessing the UI and token/notification service, by default it is http.
#It can be set to https if ssl is enabled on nginx.
ui_url_protocol = $ui_url_protocol
#The password for the root user of mysql db, change this before any production use.
db_password = $db_password
#Determine whether the UI should use compressed js files.
#For production, set it to on. For development, set it to off.
use_compressed_js = $use_compressed_js
#Maximum number of job workers in job service
max_job_workers = $max_job_workers
#Determine whether or not to generate certificate for the registry's token.
#If the value is on, the prepare script creates new root cert and private key
#for generating token to access the registry. If the value is off the default key/cert will be used.
#This flag also controls the creation of the notary signer's cert.
customize_crt = $customize_crt
#The path of cert and key files for nginx, they are applied only the protocol is set to https
ssl_cert = $ssl_cert
ssl_cert_key = $ssl_cert_key
#The path of secretkey storage
secretkey_path = /data
#Admiral's url, comment this attribute, or set its value to to NA when Harbor is standalone
admiral_url = NA
#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES
#only take effect in the first boot, the subsequent changes of these properties
#should be performed on web ui
#************************BEGIN INITIAL PROPERTIES************************
#Email account settings for sending out password resetting emails.
#Email server uses the given username and password to authenticate on TLS connections to host and act as identity.
#Identity left blank to act as username.
email_identity = $email_identity
email_server = $email_server
email_server_port = $email_server_port
email_username = $email_username
email_password = $email_password
email_from = $email_from
email_ssl = $email_ssl
##The initial password of Harbor admin, only works for the first time when Harbor starts.
#It has no effect after the first launch of Harbor.
#Change the admin password from UI after launching Harbor.
harbor_admin_password = $harbor_admin_password
##By default the auth mode is db_auth, i.e. the credentials are stored in a local database.
#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server.
auth_mode = $auth_mode
#The url for an ldap endpoint.
ldap_url = $ldap_url
#A user's DN who has the permission to search the LDAP/AD server.
#If your LDAP/AD server does not support anonymous search, you should configure this DN and ldap_search_pwd.
#ldap_searchdn = uid=searchuser,ou=people,dc=mydomain,dc=com
#the password of the ldap_searchdn
#ldap_search_pwd = password
#The base DN from which to look up a user in LDAP/AD
ldap_basedn = $ldap_basedn
#Search filter for LDAP/AD, make sure the syntax of the filter is correct.
#ldap_filter = (objectClass=person)
# The attribute used in a search to match a user, it could be uid, cn, email, sAMAccountName or other attributes depending on your LDAP/AD
ldap_uid = $ldap_uid
#the scope to search for users, 1-LDAP_SCOPE_BASE, 2-LDAP_SCOPE_ONELEVEL, 3-LDAP_SCOPE_SUBTREE
ldap_scope = $ldap_scope
#Timeout (in seconds) when connecting to an LDAP Server. The default value (and most reasonable) is 5 seconds.
ldap_timeout = 5
#Turn on or off the self-registration feature
self_registration = $self_registration
#The expiration time (in minute) of token created by token service, default is 30 minutes
token_expiration = $token_expiration
#The flag to control what users have permission to create projects
#Be default everyone can create a project, set to "adminonly" such that only admin can create project.
project_creation_restriction = $project_creation_restriction
#Determine whether the job service should verify the ssl cert when it connects to a remote registry.
#Set this flag to off when the remote registry uses a self-signed or untrusted certificate.
verify_remote_cert = $verify_remote_cert
#************************BEGIN INITIAL PROPERTIES************************
#############

View File

@ -1,117 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from string import Template
import string
import os
import sys
import argparse
from io import open
if sys.version_info[:3][0] == 2:
import ConfigParser as ConfigParser
import StringIO as StringIO
if sys.version_info[:3][0] == 3:
import configparser as ConfigParser
import io as StringIO
parser = argparse.ArgumentParser()
parser.add_argument('--source-loc', dest='source_loc', type=str,help="the path of Harbor 0.5.0 configuration file")
parser.add_argument('--source-version', dest='source_ver', type=str,help="the Harbor instance is to be deployed with notary")
parser.add_argument('--target-loc', dest='target_loc', type=str,help="the path of Harbor 1.1.x configuration file")
parser.add_argument('--target-version', dest='target_ver', type=str, help="the Harbor instance is to be deployed with notary")
upgrade_args = parser.parse_args()
# NOTE: the script only support to upgrade from 0.5.0.to 1.1.x.
def validate():
if upgrade_args.source_ver == '0.5.0' and upgrade_args.target_ver == '1.1.x':
return
raise Exception("Unable to support upgrade from %s to %s" % (upgrade_args.source_ver, upgrade_args.target_ver))
validate()
conf = StringIO.StringIO()
conf.write("[configuration]\n")
conf.write(open(upgrade_args.source_loc).read())
conf.seek(0, os.SEEK_SET)
rcp = ConfigParser.RawConfigParser()
rcp.readfp(conf)
hostname = rcp.get("configuration", "hostname")
ui_url_protocol = rcp.get("configuration", "ui_url_protocol")
email_identity = rcp.get("configuration", "email_identity")
email_server = rcp.get("configuration", "email_server")
email_server_port = rcp.get("configuration", "email_server_port")
email_username = rcp.get("configuration", "email_username")
email_password = rcp.get("configuration", "email_password")
email_from = rcp.get("configuration", "email_from")
email_ssl = rcp.get("configuration", "email_ssl")
harbor_admin_password = rcp.get("configuration", "harbor_admin_password")
auth_mode = rcp.get("configuration", "auth_mode")
ldap_url = rcp.get("configuration", "ldap_url")
ldap_basedn = rcp.get("configuration", "ldap_basedn")
ldap_uid = rcp.get("configuration", "ldap_uid")
ldap_scope = rcp.get("configuration", "ldap_scope")
db_password = rcp.get("configuration", "db_password")
self_registration = rcp.get("configuration", "self_registration")
use_compressed_js = rcp.get("configuration", "use_compressed_js")
max_job_workers = rcp.get("configuration", "max_job_workers")
token_expiration = rcp.get("configuration", "token_expiration")
verify_remote_cert = rcp.get("configuration", "verify_remote_cert")
customize_crt = rcp.get("configuration", "customize_crt")
project_creation_restriction = rcp.get("configuration", "project_creation_restriction")
ssl_cert = rcp.get("configuration", "ssl_cert")
ssl_cert_key = rcp.get("configuration", "ssl_cert_key")
def delfile(src):
if os.path.isfile(src):
try:
os.remove(src)
print("Clearing the configuration file: %s" % src)
except:
pass
elif os.path.isdir(src):
for item in os.listdir(src):
itemsrc=os.path.join(src,item)
delfile(itemsrc)
def render(src, dest, **kw):
t = Template(open(src, 'r').read())
with open(dest, 'w') as f:
f.write(t.substitute(**kw))
print("Generated configuration file: %s" % dest)
delfile(upgrade_args.target_loc)
base_dir = os.path.dirname(__file__)
config_template = os.path.join(base_dir, "harbor_1_1_0_template")
render(config_template,
upgrade_args.target_loc,
hostname=hostname,
ui_url_protocol=ui_url_protocol,
db_password=db_password,
use_compressed_js=use_compressed_js,
max_job_workers=max_job_workers,
customize_crt=customize_crt,
ssl_cert=ssl_cert,
ssl_cert_key=ssl_cert_key,
admiral_url='',
email_identity=email_identity,
email_server=email_server,
email_server_port=email_server_port,
email_username=email_username,
email_password=email_password,
email_from=email_from,
email_ssl=email_ssl,
harbor_admin_password=harbor_admin_password,
auth_mode=auth_mode,
ldap_url=ldap_url,
ldap_basedn=ldap_basedn,
ldap_uid=ldap_uid,
ldap_scope=ldap_scope,
self_registration=self_registration,
token_expiration=token_expiration,
project_creation_restriction=project_creation_restriction,
verify_remote_cert=verify_remote_cert
)

View File

@ -1,12 +1,12 @@
#!/bin/bash
export PYTHONPATH=$PYTHONPATH:/harbor-migration
export PYTHONPATH=$PYTHONPATH:/harbor-migration/db
if [ -z "$DB_USR" -o -z "$DB_PWD" ]; then
echo "DB_USR or DB_PWD not set, exiting..."
exit 1
fi
source ./alembic.tpl > ./alembic.ini
source /harbor-migration/db/alembic.tpl > /harbor-migration/db/alembic.ini
DBCNF="-hlocalhost -u${DB_USR}"
@ -23,42 +23,41 @@ if [[ $1 = "help" || $1 = "h" || $# = 0 ]]; then
exit 0
fi
if [[ ( $1 = "up" || $1 = "upgrade" ) && ${SKIP_CONFIRM} != "y" ]]; then
echo "Please backup before upgrade."
read -p "Enter y to continue updating or n to abort:" ans
case $ans in
[Yy]* )
;;
[Nn]* )
exit 0
;;
* ) echo "illegal answer: $ans. Upgrade abort!!"
exit 1
;;
esac
fi
# if [[ ( $1 = "up" || $1 = "upgrade" ) && ${SKIP_CONFIRM} != "y" ]]; then
# echo "Please backup before upgrade."
# read -p "Enter y to continue updating or n to abort:" ans
# case $ans in
# [Yy]* )
# ;;
# [Nn]* )
# exit 0
# ;;
# * ) echo "illegal answer: $ans. Upgrade abort!!"
# exit 1
# ;;
# esac
# fi
echo 'Trying to start mysql server...'
chown -R 10000:10000 /var/lib/mysql
mysqld &
echo 'Waiting for MySQL start...'
for i in {60..0}; do
mysqladmin -u$DB_USR -p$DB_PWD processlist >/dev/null 2>&1
if [ $? = 0 ]; then
break
fi
echo 'Waiting for MySQL start...'
sleep 1
done
if [ "$i" = 0 ]; then
echo "timeout. Can't run mysql server."
if [[ $1 = "test" ]]; then
echo "test failed."
echo "DB test failed."
fi
exit 1
fi
if [[ $1 = "test" ]]; then
echo "test passed."
echo "DB test passed."
exit 0
fi
@ -84,39 +83,40 @@ up|upgrade)
mysql $DBCNF -e "insert into registry.alembic_version values ('0.1.1')"
fi
fi
alembic -c ./alembic.ini current
alembic -c ./alembic.ini upgrade ${VERSION}
alembic -c /harbor-migration/db/alembic.ini current
alembic -c /harbor-migration/db/alembic.ini upgrade ${VERSION}
rc="$?"
alembic -c ./alembic.ini current
alembic -c /harbor-migration/db/alembic.ini current
echo "Upgrade performed."
echo $rc
exit $rc
;;
backup)
echo "Performing backup..."
mysqldump $DBCNF --add-drop-database --databases registry > ./backup/registry.sql
mysqldump $DBCNF --add-drop-database --databases registry > /harbor-migration/backup/registry.sql
rc="$?"
echo "Backup performed."
exit $rc
;;
export)
echo "Performing export..."
./export --dbuser ${DB_USR} --dbpwd ${DB_PWD} --exportpath ${EXPORTPATH}
/harbor-migration/db/export --dbuser ${DB_USR} --dbpwd ${DB_PWD} --exportpath ${EXPORTPATH}
rc="$?"
echo "Export performed."
echo $rc
exit $rc
;;
mapprojects)
echo "Performing map projects..."
./mapprojects --dbuser ${DB_USR} --dbpwd ${DB_PWD} --mapprojectsfile ${MAPPROJECTFILE}
/harbor-migration/db/mapprojects --dbuser ${DB_USR} --dbpwd ${DB_PWD} --mapprojectsfile ${MAPPROJECTFILE}
rc="$?"
echo "Map projects performed."
echo $rc
exit $rc
;;
restore)
echo "Performing restore..."
mysql $DBCNF < ./backup/registry.sql
mysql $DBCNF < /harbor-migration/db/backup/registry.sql
rc="$?"
echo "Restore performed."
exit $rc
;;
*)
echo "unknown option"

View File

@ -0,0 +1,6 @@
#!/bin/bash
set -e
python ./migrator.py "$@"
exec "$@"

200
tools/migration/migrator.py Normal file
View File

@ -0,0 +1,200 @@
import abc
import subprocess
from optparse import OptionParser
from shutil import copyfile
import os
import sys
import argparse
RC_VALIDATE = 101
RC_UP = 102
RC_DOWN = 103
RC_BACKUP = 104
RC_RESTORE = 105
RC_UNNKNOW_TYPE = 106
RC_GEN = 110
class DBMigrator():
def __init__(self, target):
self.target = target
self.script = "./db/run.sh"
def backup(self):
return run_cmd(self.script + " backup") == 0
def restore(self):
return run_cmd(self.script + " restore") == 0
def up(self):
cmd = self.script + " up"
if self.target != '':
cmd = cmd + " " + self.target
return run_cmd(cmd) == 0
def validate(self):
return run_cmd(self.script + " test") == 0
class CfgMigrator():
def __init__(self, target):
self.target = target
self.cfg_path = "/harbor-migration/harbor-cfg/harbor.cfg"
self.backup_path = "/harbor-migration/backup"
def backup(self):
try:
copyfile(self.cfg_path, self.backup_path+"/harbor.cfg")
print ("Success to backup harbor.cfg.")
return True
except Exception, e:
print ("Back up error: %s" % str(e))
return False
def restore(self):
if not os.path.exists(self.backup_path+"/harbor.cfg"):
print ("Unable to restore as there is no harbor.cfg")
return False
try:
copyfile(self.backup_path+"/harbor.cfg", self.cfg_path)
print ("Success to restore harbor.cfg.")
return True
except Exception, e:
print ("Restore error: %s" % str(e))
return False
def up(self):
if not os.path.exists(self.cfg_path):
print ("Skip cfg up as no harbor.cfg in the path.")
return True
cmd = "python ./cfg/run.py --input " + self.cfg_path
if self.target != '':
cmd = cmd + " --target " + self.target
return run_cmd(cmd) == 0
def validate(self):
if not os.path.exists(self.cfg_path):
print ("Unable to loacte the harbor.cfg, please check.")
return False
print ("Success to validate harbor.cfg.")
return True
class Parameters(object):
def __init__(self):
self.db_user = os.getenv('DB_USR', '')
self.db_pwd = os.getenv('DB_PWD', '')
self.skip_confirm = os.getenv('SKIP_CONFIRM', 'n')
self.is_migrate_db = True
self.is_migrate_cfg = True
self.target_version = ''
self.action = ''
self.init_from_input()
def is_action(self, action):
if action == "test" or action == "backup" or action == "restore" or action == "up":
return True
else:
return False
def parse_input(self):
argv_len = len(sys.argv[1:])
last_argv = sys.argv[argv_len:][0]
if not self.is_action(last_argv):
print ("Fail to parse input: the last parameter should in test:up:restore:backup")
sys.exit(RC_GEN)
if last_argv == 'up':
if self.skip_confirm != 'y':
if not pass_skip_confirm():
sys.exit(RC_GEN)
if argv_len == 1:
return (True, True, '', last_argv)
parser = argparse.ArgumentParser(description='migrator of harbor')
parser.add_argument('--db', action="store_true", dest='is_migrate_db', required=False, default=False, help='The flag to upgrade db.')
parser.add_argument('--cfg', action="store_true", dest='is_migrate_cfg', required=False, default=False, help='The flag to upgrede cfg.')
parser.add_argument('--version', action="store", dest='target_version', required=False, default='', help='The target version that the harbor will be migrated to.')
args = parser.parse_args(sys.argv[1:argv_len])
args.action = last_argv
return (args.is_migrate_db, args.is_migrate_cfg, args.target_version, args.action)
def init_from_input(self):
(self.is_migrate_db, self.is_migrate_cfg, self.target_version, self.action) = self.parse_input()
def run_cmd(cmd):
return os.system(cmd)
def pass_skip_confirm():
valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
message = "Please backup before upgrade, \nEnter y to continue updating or n to abort: "
while True:
sys.stdout.write(message)
choice = raw_input().lower()
if choice == '':
return False
elif choice in valid:
return valid[choice]
else:
sys.stdout.write("Please respond with 'yes' or 'no' "
"(or 'y' or 'n').\n")
def main():
commandline_input = Parameters()
db_migrator = DBMigrator(commandline_input.target_version)
cfg_migrator = CfgMigrator(commandline_input.target_version)
try:
# test
if commandline_input.action == "test":
if commandline_input.is_migrate_db:
if not db_migrator.validate():
print ("Fail to validate: please make sure your DB auth is correct.")
sys.exit(RC_VALIDATE)
if commandline_input.is_migrate_cfg:
if not cfg_migrator.validate():
print ("Fail to validate: please make sure your cfg path is correct.")
sys.exit(RC_VALIDATE)
# backup
elif commandline_input.action == "backup":
if commandline_input.is_migrate_db:
if not db_migrator.backup():
sys.exit(RC_BACKUP)
if commandline_input.is_migrate_cfg:
if not cfg_migrator.backup():
sys.exit(RC_BACKUP)
# up
elif commandline_input.action == "up":
if commandline_input.is_migrate_db:
if not db_migrator.up():
sys.exit(RC_UP)
if commandline_input.is_migrate_cfg:
if not cfg_migrator.up():
sys.exit(RC_UP)
# restore
elif commandline_input.action == "restore":
if commandline_input.is_migrate_db:
if not db_migrator.restore():
sys.exit(RC_RESTORE)
if commandline_input.is_migrate_cfg:
if not cfg_migrator.restore():
sys.exit(RC_RESTORE)
else:
print ("Unknow action type: " + str(commandline_input.action))
sys.exit(RC_UNNKNOW_TYPE)
except Exception as ex:
print ("Migrator fail to execute, err: " + ex.message)
sys.exit(RC_GEN)
if __name__ == '__main__':
main()