diff --git a/make/harbor.cfg b/make/harbor.cfg index f784947f9..9e8582224 100644 --- a/make/harbor.cfg +++ b/make/harbor.cfg @@ -1,5 +1,7 @@ ## Configuration file of Harbor +#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! +_version = 1.5.0 #The IP address or hostname to access admin UI and registry service. #DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. hostname = reg.mydomain.com diff --git a/tools/migration/cfg/migrate.py b/tools/migration/cfg/migrate.py new file mode 100644 index 000000000..25c01a091 --- /dev/null +++ b/tools/migration/cfg/migrate.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + + +from __future__ import print_function +import argparse +import os +import utils +import importlib +import glob +import shutil + +def main(): + target_version = '1.5.0' + parser = argparse.ArgumentParser(description='migrator of harbor.cfg') + parser.add_argument('--input', '-i', action="store", dest='input_path', required=True, help='The path to the old harbor.cfg that provides input value, this required value') + parser.add_argument('--output','-o', action="store", dest='output_path', required=False, help='The path of the migrated harbor.cfg, if not set the input file will be overwritten') + parser.add_argument('--target', action="store", dest='target_version', help='The target version that the harbor.cfg will be migrated to.') + args = parser.parse_args() + if args.output_path is None: + args.output_path = args.input_path + if args.target_version is not None: + target_version = args.target_version + input_version = utils.get_conf_version(args.input_path) + curr_dir = os.path.dirname(__file__) + chain = [] + if not search(curr_dir, input_version, target_version, chain): + print ("No migrator for version: %s" % input_version) + os.exit(1) + else: + print ("input version: %s, migrator chain: %s" % (input_version, chain)) + curr_input_path = args.input_path + for c in chain: + #TODO: more real-world testing needed for chained-migration. + m = importlib.import_module(to_module_path(c)) + curr_output_path = "harbor.cfg.%s.tmp" % c + print("migrating to version %s" % c) + m.migrate(curr_input_path, curr_output_path) + curr_input_path = curr_output_path + shutil.copy(curr_output_path, args.output_path) + print("Written new values to %s" % args.output_path) + for tmp_f in glob.glob("harbor.cfg.*.tmp"): + os.remove(tmp_f) + +def to_module_path(ver): + return "migrator_%s" % ver.replace(".","_") + +def search(basedir, input_ver, target_ver, l): + module = to_module_path(target_ver) + if os.path.isdir(os.path.join(basedir, module)): + m = importlib.import_module(module) + if input_ver in m.acceptable_versions: + l.append(target_ver) + return True + for v in m.acceptable_versions: + if search(basedir, input_ver, v, l): + l.append(target_ver) + return True + return False + + +if __name__ == "__main__": + main() diff --git a/tools/migration/cfg/migrator_1_5_0/__init__.py b/tools/migration/cfg/migrator_1_5_0/__init__.py new file mode 100644 index 000000000..a239df0a1 --- /dev/null +++ b/tools/migration/cfg/migrator_1_5_0/__init__.py @@ -0,0 +1,41 @@ +from __future__ import print_function +import utils +import os +acceptable_versions = ['1.2.0', '1.3.0', '1.4.0'] + +#In 1.5 template the placeholder has the same value as the attribute name. +default = { + 'log_rotate_count':'50', + 'log_rotate_size':'200M', + 'db_host':'mysql', + 'db_port':'3306', + 'db_user':'root', + 'redis_url':'', + 'clair_db_host':'postgres', + 'clair_db_port':'5432', + 'clair_db_username':'postgres', + 'clair_db':'postgres', + 'uaa_endpoint':'uaa.mydomain.org', + 'uaa_clientid':'id', + 'uaa_clientsecret':'secret', + 'uaa_verify_cert':'true', + 'uaa_ca_cert':'/path/to/ca.pem', + 'registry_storage_provider_name':'filesystem', + 'registry_storage_provider_config':'' +} + +def migrate(input_cfg, output_cfg): + d = utils.read_conf(input_cfg) + keys = default.keys() + keys.extend(['hostname', 'ui_url_protocol', 'max_job_workers', 'customize_crt', + 'ssl_cert', 'ssl_cert_key', 'secretkey_path', 'admiral_url', 'db_password', 'clair_db_password']) + val = {} + for k in keys: + if d.has_key(k): + val[k] = d[k] + else: + val[k] = default[k] + tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.cfg.tpl') + utils.render(tpl_path, output_cfg, **val) + + diff --git a/tools/migration/cfg/migrator_1_5_0/harbor.cfg.tpl b/tools/migration/cfg/migrator_1_5_0/harbor.cfg.tpl new file mode 100644 index 000000000..1a0be1b53 --- /dev/null +++ b/tools/migration/cfg/migrator_1_5_0/harbor.cfg.tpl @@ -0,0 +1,161 @@ +## Configuration file of Harbor + +#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! +_version = 1.5.0 +#The IP address or hostname to access admin UI and registry service. +#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. +hostname = $hostname + +#The protocol for accessing the UI and token/notification service, by default it is http. +#It can be set to https if ssl is enabled on nginx. +ui_url_protocol = $ui_url_protocol + +#Maximum number of job workers in job service +max_job_workers = $max_job_workers + +#Determine whether or not to generate certificate for the registry's token. +#If the value is on, the prepare script creates new root cert and private key +#for generating token to access the registry. If the value is off the default key/cert will be used. +#This flag also controls the creation of the notary signer's cert. +customize_crt = $customize_crt + +#The path of cert and key files for nginx, they are applied only the protocol is set to https +ssl_cert = $ssl_cert +ssl_cert_key = $ssl_cert_key + +#The path of secretkey storage +secretkey_path = $secretkey_path + +#Admiral's url, comment this attribute, or set its value to NA when Harbor is standalone +admiral_url = $admiral_url + +#Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. +log_rotate_count = $log_rotate_count +#Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. +#If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G +#are all valid. +log_rotate_size = $log_rotate_size + +#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES +#only take effect in the first boot, the subsequent changes of these properties +#should be performed on web ui + +#************************BEGIN INITIAL PROPERTIES************************ + +#Email account settings for sending out password resetting emails. + +#Email server uses the given username and password to authenticate on TLS connections to host and act as identity. +#Identity left blank to act as username. +email_identity = + +email_server = smtp.mydomain.com +email_server_port = 25 +email_username = sample_admin@mydomain.com +email_password = abc +email_from = admin +email_ssl = false +email_insecure = false + +##The initial password of Harbor admin, only works for the first time when Harbor starts. +#It has no effect after the first launch of Harbor. +#Change the admin password from UI after launching Harbor. +harbor_admin_password = Harbor12345 + +##By default the auth mode is db_auth, i.e. the credentials are stored in a local database. +#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server. +auth_mode = db_auth + +#The url for an ldap endpoint. +ldap_url = ldaps://ldap.mydomain.com + +#A user's DN who has the permission to search the LDAP/AD server. +#If your LDAP/AD server does not support anonymous search, you should configure this DN and ldap_search_pwd. +#ldap_searchdn = uid=searchuser,ou=people,dc=mydomain,dc=com + +#the password of the ldap_searchdn +#ldap_search_pwd = password + +#The base DN from which to look up a user in LDAP/AD +ldap_basedn = ou=people,dc=mydomain,dc=com + +#Search filter for LDAP/AD, make sure the syntax of the filter is correct. +#ldap_filter = (objectClass=person) + +# The attribute used in a search to match a user, it could be uid, cn, email, sAMAccountName or other attributes depending on your LDAP/AD +ldap_uid = uid + +#the scope to search for users, 0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE +ldap_scope = 2 + +#Timeout (in seconds) when connecting to an LDAP Server. The default value (and most reasonable) is 5 seconds. +ldap_timeout = 5 + +#Verify certificate from LDAP server +ldap_verify_cert = true + +#Turn on or off the self-registration feature +self_registration = on + +#The expiration time (in minute) of token created by token service, default is 30 minutes +token_expiration = 30 + +#The flag to control what users have permission to create projects +#The default value "everyone" allows everyone to creates a project. +#Set to "adminonly" so that only admin user can create project. +project_creation_restriction = everyone + +#************************END INITIAL PROPERTIES************************ + +#######Harbor DB configuration section####### + +#The address of the Harbor database. Only need to change when using external db. +db_host = $db_host + +#The password for the root user of Harbor DB. Change this before any production use. +db_password = $db_password + +#The port of Harbor database host +db_port = $db_port + +#The user name of Harbor database +db_user = $db_user + +##### End of Harbor DB configuration####### + +#The redis server address. Only needed in HA installation. +redis_url = $redis_url + +##########Clair DB configuration############ + +#Clair DB host address. Only change it when using an exteral DB. +clair_db_host = $clair_db_host + +#The password of the Clair's postgres database. Only effective when Harbor is deployed with Clair. +#Please update it before deployment. Subsequent update will cause Clair's API server and Harbor unable to access Clair's database. +clair_db_password = $clair_db_password + +#Clair DB connect port +clair_db_port = $clair_db_port + +#Clair DB username +clair_db_username = $clair_db_username + +#Clair default database +clair_db = $clair_db + +##########End of Clair DB configuration############ + +#The following attributes only need to be set when auth mode is uaa_auth +uaa_endpoint = $uaa_endpoint +uaa_clientid = $uaa_clientid +uaa_clientsecret = $uaa_clientsecret +uaa_verify_cert = $uaa_verify_cert +uaa_ca_cert = $uaa_ca_cert + + +### Docker Registry setting ### +#registry_storage_provider can be: filesystem, s3, gcs, azure, etc. +registry_storage_provider_name = $registry_storage_provider_name +#registry_storage_provider_config is a comma separated "key: value" pairs, e.g. "key1: value, key2: value2". +#Refer to https://docs.docker.com/registry/configuration/#storage for all available configuration. +registry_storage_provider_config = $registry_storage_provider_config diff --git a/tools/migration/cfg/utils.py b/tools/migration/cfg/utils.py new file mode 100644 index 000000000..5996012c5 --- /dev/null +++ b/tools/migration/cfg/utils.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import sys +import os +import json +from string import Template + +if sys.version_info[:3][0] == 2: + import ConfigParser as ConfigParser + import StringIO as StringIO + +if sys.version_info[:3][0] == 3: + import configparser as ConfigParser + import io as StringIO + +def read_conf(path): + temp_section = "configuration" + conf = StringIO.StringIO() + conf.write("[%s]\n" % temp_section) + conf.write(open(path).read()) + conf.seek(0, os.SEEK_SET) + rcp = ConfigParser.RawConfigParser() + rcp.readfp(conf) + d = {} + for op in rcp.options(temp_section): + d[op] = rcp.get(temp_section, op) + return d + +def get_conf_version(path): + d = read_conf(path) +# print json.dumps(d,indent=4) + if d.has_key("_version"): # >=1.5.0 + return d["_version"] + if not d.has_key("clair_db_password"): + return "unsupported" + if d.has_key("registry_storage_provider_name"): + return "1.4.0" + if d.has_key("uaa_endpoint"): + return "1.3.0" + return "1.2.0" + +def render(src, dest, **kw): + t = Template(open(src, 'r').read()) + with open(dest, 'w') as f: + f.write(t.substitute(**kw)) + + + + + + diff --git a/tools/migration/Dockerfile b/tools/migration/db/Dockerfile similarity index 100% rename from tools/migration/Dockerfile rename to tools/migration/db/Dockerfile diff --git a/tools/migration/alembic.sql b/tools/migration/db/alembic.sql similarity index 100% rename from tools/migration/alembic.sql rename to tools/migration/db/alembic.sql diff --git a/tools/migration/alembic.tpl b/tools/migration/db/alembic.tpl similarity index 100% rename from tools/migration/alembic.tpl rename to tools/migration/db/alembic.tpl diff --git a/tools/migration/changelog.md b/tools/migration/db/changelog.md similarity index 100% rename from tools/migration/changelog.md rename to tools/migration/db/changelog.md diff --git a/tools/migration/db_meta.py b/tools/migration/db/db_meta.py similarity index 100% rename from tools/migration/db_meta.py rename to tools/migration/db/db_meta.py diff --git a/tools/migration/export b/tools/migration/db/export similarity index 100% rename from tools/migration/export rename to tools/migration/db/export diff --git a/tools/migration/import b/tools/migration/db/import similarity index 100% rename from tools/migration/import rename to tools/migration/db/import diff --git a/tools/migration/mapprojects b/tools/migration/db/mapprojects similarity index 100% rename from tools/migration/mapprojects rename to tools/migration/db/mapprojects diff --git a/tools/migration/migration_cfg/harbor_1_1_0_template b/tools/migration/db/migration_cfg/harbor_1_1_0_template similarity index 100% rename from tools/migration/migration_cfg/harbor_1_1_0_template rename to tools/migration/db/migration_cfg/harbor_1_1_0_template diff --git a/tools/migration/migration_cfg/upgrade b/tools/migration/db/migration_cfg/upgrade similarity index 100% rename from tools/migration/migration_cfg/upgrade rename to tools/migration/db/migration_cfg/upgrade diff --git a/tools/migration/migration_harbor/env.py b/tools/migration/db/migration_harbor/env.py similarity index 100% rename from tools/migration/migration_harbor/env.py rename to tools/migration/db/migration_harbor/env.py diff --git a/tools/migration/migration_harbor/script.py.mako b/tools/migration/db/migration_harbor/script.py.mako similarity index 100% rename from tools/migration/migration_harbor/script.py.mako rename to tools/migration/db/migration_harbor/script.py.mako diff --git a/tools/migration/migration_harbor/versions/0_1_1.py b/tools/migration/db/migration_harbor/versions/0_1_1.py similarity index 100% rename from tools/migration/migration_harbor/versions/0_1_1.py rename to tools/migration/db/migration_harbor/versions/0_1_1.py diff --git a/tools/migration/migration_harbor/versions/0_3_0.py b/tools/migration/db/migration_harbor/versions/0_3_0.py similarity index 100% rename from tools/migration/migration_harbor/versions/0_3_0.py rename to tools/migration/db/migration_harbor/versions/0_3_0.py diff --git a/tools/migration/migration_harbor/versions/0_4_0.py b/tools/migration/db/migration_harbor/versions/0_4_0.py similarity index 100% rename from tools/migration/migration_harbor/versions/0_4_0.py rename to tools/migration/db/migration_harbor/versions/0_4_0.py diff --git a/tools/migration/migration_harbor/versions/1_2_0.py b/tools/migration/db/migration_harbor/versions/1_2_0.py similarity index 100% rename from tools/migration/migration_harbor/versions/1_2_0.py rename to tools/migration/db/migration_harbor/versions/1_2_0.py diff --git a/tools/migration/migration_harbor/versions/1_3_0.py b/tools/migration/db/migration_harbor/versions/1_3_0.py similarity index 100% rename from tools/migration/migration_harbor/versions/1_3_0.py rename to tools/migration/db/migration_harbor/versions/1_3_0.py diff --git a/tools/migration/migration_harbor/versions/1_4_0.py b/tools/migration/db/migration_harbor/versions/1_4_0.py similarity index 100% rename from tools/migration/migration_harbor/versions/1_4_0.py rename to tools/migration/db/migration_harbor/versions/1_4_0.py diff --git a/tools/migration/run.sh b/tools/migration/db/run.sh similarity index 100% rename from tools/migration/run.sh rename to tools/migration/db/run.sh