harbor/make/prepare

#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import print_function, unicode_literals # We require Python 2.6 or later
from string import Template
import random
import string
import os
import sys
import argparse
import subprocess
import shutil
from io import open

if sys.version_info[:3][0] == 2:
    import ConfigParser as ConfigParser
    import StringIO as StringIO

if sys.version_info[:3][0] == 3:
    import configparser as ConfigParser
    import io as StringIO

def validate(conf, args): 
    protocol = rcp.get("configuration", "ui_url_protocol")
    if protocol != "https" and args.notary_mode:
        raise Exception("Error: the protocol must be https when Harbor is deployed with Notary")
    if protocol == "https":
        if not rcp.has_option("configuration", "ssl_cert"):
            raise Exception("Error: The protocol is https but attribute ssl_cert is not set")
        cert_path = rcp.get("configuration", "ssl_cert")
        if not os.path.isfile(cert_path):
            raise Exception("Error: The path for certificate: %s is invalid" % cert_path)
        if not rcp.has_option("configuration", "ssl_cert_key"):
            raise Exception("Error: The protocol is https but attribute ssl_cert_key is not set")
        cert_key_path = rcp.get("configuration", "ssl_cert_key")
        if not os.path.isfile(cert_key_path):
            raise Exception("Error: The path for certificate key: %s is invalid" % cert_key_path)
    project_creation = rcp.get("configuration", "project_creation_restriction")

    if project_creation != "everyone" and project_creation != "adminonly":
        raise Exception("Error invalid value for project_creation_restriction: %s" % project_creation)

def get_secret_key(path):
    key_file = os.path.join(path, "secretkey")
    if os.path.isfile(key_file):
        with open(key_file, 'r') as f:
            key = f.read()
            print("loaded secret key")
        if len(key) != 16:
            raise Exception("secret key's length has to be 16 chars, current length: %d" % len(key))
        return key
    if not os.path.isdir(path):
        os.makedirs(path, mode=0600)
    key = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16))  
    with open(key_file, 'w') as f:
        f.write(key)
        print("generated and saved secret key")
    return key

def prep_conf_dir(root, name):
    absolute_path = os.path.join(root, name)
    if not os.path.exists(absolute_path):
        os.makedirs(absolute_path)
    return absolute_path

def render(src, dest, **kw):
    t = Template(open(src, 'r').read())
    with open(dest, 'w') as f:
        f.write(t.substitute(**kw))
    print("Generated configuration file: %s" % dest)

base_dir = os.path.dirname(__file__)
config_dir = os.path.join(base_dir, "common/config")
templates_dir = os.path.join(base_dir, "common/templates")
def delfile(src):
    if os.path.isfile(src):
        try:
            os.remove(src)
            print("Clearing the configuration file: %s" % src)
        except:
            pass
    elif os.path.isdir(src):
        for item in os.listdir(src):
            itemsrc=os.path.join(src,item)
            delfile(itemsrc)

parser = argparse.ArgumentParser()
parser.add_argument('--conf', dest='cfgfile', default=base_dir+'/harbor.cfg',type=str,help="the path of Harbor configuration file")
parser.add_argument('--with-notary', dest='notary_mode', default=False, action='store_true', help="the Harbor instance is to be deployed with notary")
args = parser.parse_args()

delfile(config_dir)
#Read configurations
conf = StringIO.StringIO()
conf.write("[configuration]\n")
conf.write(open(args.cfgfile).read())
conf.seek(0, os.SEEK_SET)
rcp = ConfigParser.RawConfigParser()
rcp.readfp(conf)

validate(rcp, args)

hostname = rcp.get("configuration", "hostname")
protocol = rcp.get("configuration", "ui_url_protocol")
ui_url = protocol + "://" + hostname
email_identity = rcp.get("configuration", "email_identity")
email_host = rcp.get("configuration", "email_server")
email_port = rcp.get("configuration", "email_server_port")
email_usr = rcp.get("configuration", "email_username")
email_pwd = rcp.get("configuration", "email_password")
email_from = rcp.get("configuration", "email_from")
email_ssl = rcp.get("configuration", "email_ssl")
harbor_admin_password = rcp.get("configuration", "harbor_admin_password")
auth_mode = rcp.get("configuration", "auth_mode")
ldap_url = rcp.get("configuration", "ldap_url")
# this two options are either both set or unset
if rcp.has_option("configuration", "ldap_searchdn"):
    ldap_searchdn = rcp.get("configuration", "ldap_searchdn")
    ldap_search_pwd = rcp.get("configuration", "ldap_search_pwd")
else:
    ldap_searchdn = ""
    ldap_search_pwd = ""
ldap_basedn = rcp.get("configuration", "ldap_basedn")
# ldap_filter is null by default
if rcp.has_option("configuration", "ldap_filter"):
    ldap_filter = rcp.get("configuration", "ldap_filter")
else:
    ldap_filter = ""
ldap_uid = rcp.get("configuration", "ldap_uid")
ldap_scope = rcp.get("configuration", "ldap_scope")
ldap_timeout = rcp.get("configuration", "ldap_timeout")
db_password = rcp.get("configuration", "db_password")
self_registration = rcp.get("configuration", "self_registration")
use_compressed_js = rcp.get("configuration", "use_compressed_js")
if protocol == "https":
    cert_path = rcp.get("configuration", "ssl_cert")
    cert_key_path = rcp.get("configuration", "ssl_cert_key")
customize_crt = rcp.get("configuration", "customize_crt")
crt_country = rcp.get("configuration", "crt_country")
crt_state = rcp.get("configuration", "crt_state")
crt_location = rcp.get("configuration", "crt_location")
crt_organization = rcp.get("configuration", "crt_organization")
crt_organizationalunit = rcp.get("configuration", "crt_organizationalunit")
crt_commonname = rcp.get("configuration", "crt_commonname")
crt_email = rcp.get("configuration", "crt_email")
max_job_workers = rcp.get("configuration", "max_job_workers")
token_expiration = rcp.get("configuration", "token_expiration")
verify_remote_cert = rcp.get("configuration", "verify_remote_cert")
proj_cre_restriction = rcp.get("configuration", "project_creation_restriction")
secretkey_path = rcp.get("configuration", "secretkey_path")
if rcp.has_option("configuration", "admiral_url"):
    admiral_url = rcp.get("configuration", "admiral_url")
else:
    admiral_url = ""
secret_key = get_secret_key(secretkey_path)
########

ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16))  
jobservice_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16))  

adminserver_config_dir = os.path.join(config_dir,"adminserver")
if not os.path.exists(adminserver_config_dir):
    os.makedirs(os.path.join(config_dir, "adminserver"))

ui_config_dir = prep_conf_dir(config_dir,"ui")
db_config_dir = prep_conf_dir(config_dir, "db")
job_config_dir = prep_conf_dir(config_dir, "jobservice")
registry_config_dir = prep_conf_dir(config_dir, "registry")
nginx_config_dir = prep_conf_dir (config_dir, "nginx")
nginx_conf_d = prep_conf_dir(nginx_config_dir, "conf.d")

adminserver_conf_env = os.path.join(config_dir, "adminserver", "env")
ui_conf_env = os.path.join(config_dir, "ui", "env")
ui_conf = os.path.join(config_dir, "ui", "app.conf")
jobservice_conf = os.path.join(config_dir, "jobservice", "app.conf")
registry_conf = os.path.join(config_dir, "registry", "config.yml")
db_conf_env = os.path.join(config_dir, "db", "env")
job_conf_env = os.path.join(config_dir, "jobservice", "env")
nginx_conf = os.path.join(config_dir, "nginx", "nginx.conf")
cert_dir = os.path.join(config_dir, "nginx", "cert") 

if protocol == "https":
    target_cert_path = os.path.join(cert_dir, os.path.basename(cert_path))
    if not os.path.exists(cert_dir):
        os.makedirs(cert_dir)
    shutil.copy2(cert_path,target_cert_path)
    target_cert_key_path = os.path.join(cert_dir, os.path.basename(cert_key_path))
    shutil.copy2(cert_key_path,target_cert_key_path)
    render(os.path.join(templates_dir, "nginx", "nginx.https.conf"),
            nginx_conf,
            ssl_cert = os.path.join("/etc/nginx/cert", os.path.basename(target_cert_path)),
            ssl_cert_key = os.path.join("/etc/nginx/cert", os.path.basename(target_cert_key_path)))
else:
    render(os.path.join(templates_dir, "nginx", "nginx.http.conf"),
        nginx_conf)
		
render(os.path.join(templates_dir, "adminserver", "env"),
        adminserver_conf_env,
        ui_url=ui_url,
        auth_mode=auth_mode,
        self_registration=self_registration,
        ldap_url=ldap_url,
        ldap_searchdn =ldap_searchdn, 
        ldap_search_pwd =ldap_search_pwd,
        ldap_basedn=ldap_basedn,
        ldap_filter=ldap_filter,
        ldap_uid=ldap_uid,
        ldap_scope=ldap_scope,
        ldap_timeout=ldap_timeout,
        db_password=db_password,
        email_host=email_host,
        email_port=email_port,
        email_usr=email_usr,
        email_pwd=email_pwd,
        email_ssl=email_ssl,
        email_from=email_from,
        email_identity=email_identity,
        harbor_admin_password=harbor_admin_password,
        project_creation_restriction=proj_cre_restriction,
        verify_remote_cert=verify_remote_cert,
        max_job_workers=max_job_workers,
        ui_secret=ui_secret,
        jobservice_secret=jobservice_secret,
        token_expiration=token_expiration,
        admiral_url=admiral_url,
        with_notary=args.notary_mode,
        use_compressed_js=use_compressed_js
	)

render(os.path.join(templates_dir, "ui", "env"), 
        ui_conf_env, 
        ui_secret=ui_secret,
        jobservice_secret=jobservice_secret,)

render(os.path.join(templates_dir, "registry", 
		"config.yml"),
        registry_conf,
        ui_url=ui_url)

render(os.path.join(templates_dir, "db", "env"),
        db_conf_env,
        db_password=db_password)

render(os.path.join(templates_dir, "jobservice", "env"),
        job_conf_env,
        ui_secret=ui_secret,
        jobservice_secret=jobservice_secret)

print("Generated configuration file: %s" % jobservice_conf)
shutil.copyfile(os.path.join(templates_dir, "jobservice", "app.conf"), jobservice_conf)

print("Generated configuration file: %s" % ui_conf)
shutil.copyfile(os.path.join(templates_dir, "ui", "app.conf"), ui_conf)


def validate_crt_subj(dirty_subj):
    subj_list = [item for item in dirty_subj.strip().split("/") \
        if len(item.split("=")) == 2 and len(item.split("=")[1]) > 0]
    return "/" + "/".join(subj_list)

FNULL = open(os.devnull, 'w')

from functools import wraps
def stat_decorator(func):
    @wraps(func)
    def check_wrapper(*args, **kwargs):
        stat = func(*args, **kwargs)
        message = "Generated configuration file: %s" % kwargs['path'] \
                if stat == 0 else "Fail to generate %s" % kwargs['path']
        print(message)
        if stat != 0:
            sys.exit(1)
    return check_wrapper

@stat_decorator
def check_private_key_stat(*args, **kwargs):
    return subprocess.call(["openssl", "genrsa", "-out", kwargs['path'], "4096"],\
        stdout=FNULL, stderr=subprocess.STDOUT)

@stat_decorator
def check_certificate_stat(*args, **kwargs):
    dirty_subj = "/C={0}/ST={1}/L={2}/O={3}/OU={4}/CN={5}/emailAddress={6}"\
        .format(crt_country, crt_state, crt_location, crt_organization,\
        crt_organizationalunit, crt_commonname, crt_email)
    subj = validate_crt_subj(dirty_subj)
    return subprocess.call(["openssl", "req", "-new", "-x509", "-key",\
        private_key_pem, "-out", root_crt, "-days", "3650", "-subj", subj], \
        stdout=FNULL, stderr=subprocess.STDOUT)

def openssl_is_installed(stat):
    if stat == 0:
        return True
    else:
        print("Cannot find openssl installed in this computer\nUse default SSL certificate file")
        return False

if customize_crt == 'on':
    shell_stat = subprocess.check_call(["which", "openssl"], stdout=FNULL, stderr=subprocess.STDOUT)
    if openssl_is_installed(shell_stat):
        private_key_pem = os.path.join(config_dir, "ui", "private_key.pem")
        root_crt = os.path.join(config_dir, "registry", "root.crt")

        check_private_key_stat(path=private_key_pem)
        check_certificate_stat(path=root_crt)
else:
    print("Generated configuration file: %s" % ui_config_dir + "private_key.pem")
    shutil.copyfile(os.path.join(templates_dir, "ui", "private_key.pem"), os.path.join(ui_config_dir, "private_key.pem"))
    print("Generated configuration file: %s" % registry_config_dir + "root.crt")
    shutil.copyfile(os.path.join(templates_dir, "registry", "root.crt"), os.path.join(registry_config_dir, "root.crt"))
	
FNULL.close()
if args.notary_mode:
    notary_config_dir = prep_conf_dir(config_dir, "notary")
    notary_temp_dir = os.path.join(templates_dir, "notary") 
    print("Copying sql file for notary DB")
    if os.path.exists(os.path.join(notary_config_dir, "mysql-initdb.d")):
        shutil.rmtree(os.path.join(notary_config_dir, "mysql-initdb.d"))
    shutil.copytree(os.path.join(notary_temp_dir, "mysql-initdb.d"), os.path.join(notary_config_dir, "mysql-initdb.d")) 
    #TODO:generate certs?
    print("Copying certs for notary signer")
    shutil.copy2(os.path.join(notary_temp_dir, "notary-signer.crt"), notary_config_dir)
    shutil.copy2(os.path.join(notary_temp_dir, "notary-signer.key"), notary_config_dir)
    shutil.copy2(os.path.join(notary_temp_dir, "root-ca.crt"), notary_config_dir)

    shutil.copy2(os.path.join(registry_config_dir, "root.crt"), notary_config_dir)
    print("Copying notary signer configuration file")
    shutil.copy2(os.path.join(notary_temp_dir, "signer-config.json"), notary_config_dir)
    render(os.path.join(notary_temp_dir, "server-config.json"),
        os.path.join(notary_config_dir, "server-config.json"),
        token_endpoint=ui_url)

    print("Copying nginx configuration file for notary")
    shutil.copy2(os.path.join(templates_dir, "nginx", "notary.upstream.conf"), nginx_conf_d)
    shutil.copy2(os.path.join(templates_dir, "nginx", "notary.location.conf"), nginx_conf_d)

    default_alias = ''.join(random.choice(string.ascii_letters) for i in range(8))
    render(os.path.join(notary_temp_dir, "signer_env"), os.path.join(notary_config_dir, "signer_env"), alias = default_alias)

		
print("The configuration files are ready, please use docker-compose to start the service.")