diff --git a/Makefile b/Makefile index 93787f395..d238cf9ab 100644 --- a/Makefile +++ b/Makefile @@ -295,7 +295,7 @@ compile: check_environment versions_prepare compile_core compile_jobservice comp update_prepare_version: @echo "substitude the prepare version tag in prepare file..." - $(SEDCMD) -i -e 's/goharbor\/prepare:.*[[:space:]]\+/goharbor\/prepare:$(VERSIONTAG) /' $(MAKEPATH)/prepare ; + @$(SEDCMD) -i -e 's/goharbor\/prepare:.*[[:space:]]\+/goharbor\/prepare:$(VERSIONTAG) /' $(MAKEPATH)/prepare ; prepare: update_prepare_version @echo "preparing..." @@ -310,7 +310,7 @@ build: install: compile ui_version build prepare start -package_online: prepare +package_online: update_prepare_version @echo "packing online package ..." @cp -r make $(HARBORPKG) @if [ -n "$(REGISTRYSERVER)" ] ; then \ diff --git a/make/harbor.yml b/make/harbor.yml index 7f7bf93e8..7abb44217 100644 --- a/make/harbor.yml +++ b/make/harbor.yml @@ -9,7 +9,7 @@ http: # https: # port: 443 -# #The path of cert and key files for nginx, they are applied only the protocol is set to https +# #The path of cert and key files for nginx # certificate: /your/certificate/path # private_key: /your/private/key/path @@ -30,64 +30,19 @@ database: # The default data volume data_volume: /data -# Harbor Storage settings -storage_service: - # registry_custom_ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore - # of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signed certificate. - ca_bundle: +# Harbor Storage settings by default is using /data dir on local filesystem +# Uncomment storage_service setting If you want to using external storage +# storage_service: +# # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore +# # of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signed certificate. +# ca_bundle: - # storaged in $data_volume/ - # filesystem: - # maxthreads: 100 - # azure: - # accountname: accountname - # accountkey: base64encodedaccountkey - # container: containername - # gcs: - # bucket: bucketname - # keyfile: /path/to/keyfile - # rootdirectory: /gcs/object/name/prefix - # chunksize: 5242880 - # s3: - # accesskey: awsaccesskey - # secretkey: awssecretkey - # region: us-west-1 - # regionendpoint: http://myobjects.local - # bucket: bucketname - # encrypt: true - # keyid: mykeyid - # secure: true - # v4auth: true - # chunksize: 5242880 - # multipartcopychunksize: 33554432 - # multipartcopymaxconcurrency: 100 - # multipartcopythresholdsize: 33554432 - # rootdirectory: /s3/object/name/prefix - # swift: - # username: username - # password: password - # authurl: https://storage.myprovider.com/auth/v1.0 or https://storage.myprovider.com/v2.0 or https://storage.myprovider.com/v3/auth - # tenant: tenantname - # tenantid: tenantid - # domain: domain name for Openstack Identity v3 API - # domainid: domain id for Openstack Identity v3 API - # insecureskipverify: true - # region: fr - # container: containername - # rootdirectory: /swift/object/name/prefix - # oss: - # accesskeyid: accesskeyid - # accesskeysecret: accesskeysecret - # region: OSS region name - # endpoint: optional endpoints - # internal: optional internal endpoint - # bucket: OSS bucket - # encrypt: optional data encryption setting - # secure: optional ssl setting - # chunksize: optional size valye - # rootdirectory: optional root directory +# # storage backend, default is filesystem, options include filesystem, azure, gcs, s3, swift and oss +# # for more info about this configuration please refer https://docs.docker.com/registry/configuration/ +# filesystem: +# maxthreads: 100 -# Clair DB configuration +# Clair configuration clair: # The interval of clair updaters, the unit is hour, set to 0 to disable the updaters. updaters_interval: 12 @@ -118,7 +73,8 @@ log: #This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! _version: 1.7.0 -# Uncomment if using external database. Only support postgres. +# Uncomment external_database if using external database. And the password will replace the the password setting in database. +# And currently ontly support postgres. # external_database: # host: postgresql # port: 5432 @@ -126,7 +82,7 @@ _version: 1.7.0 # password: root123 # ssl_mode: disable -# Umcomments if using external Redis server +# Umcomments external_redis if using external Redis server # external_redis: # host: redis # port: 6379 @@ -134,4 +90,4 @@ _version: 1.7.0 # # db_index 0 is for core, it's unchangeable # registry_db_index: 1 # jobservice_db_index: 2 -# chartmuseum_db_index: 3 \ No newline at end of file +# chartmuseum_db_index: 3 diff --git a/make/install.sh b/make/install.sh index 25247c951..b01da719c 100755 --- a/make/install.sh +++ b/make/install.sh @@ -192,9 +192,9 @@ docker-compose up -d protocol=http hostname=reg.mydomain.com -if [[ $(cat ./harbor.yml) =~ ui_url_protocol:[[:blank:]]*(https?) ]] +if [ -n "$(grep '^[^#]*https:' ./harbor.yml)"] then -protocol=${BASH_REMATCH[1]} +protocol=https fi if [[ $(grep '^[[:blank:]]*hostname:' ./harbor.yml) =~ hostname:[[:blank:]]*(.*) ]] diff --git a/make/photon/prepare/g.py b/make/photon/prepare/g.py index 810816781..bb766f07b 100644 --- a/make/photon/prepare/g.py +++ b/make/photon/prepare/g.py @@ -20,10 +20,8 @@ private_key_pem_path = Path('/secret/core/private_key.pem') root_crt_path = Path('/secret/registry/root.crt') config_file_path = '/compose_location/harbor.yml' +input_config_path = '/input/harbor.yml' versions_file_path = Path('/usr/src/app/versions') cert_dir = os.path.join(config_dir, "nginx", "cert") -core_cert_dir = os.path.join(config_dir, "core", "certificates") - -registry_custom_ca_bundle_storage_path = Path('/secret/common/custom-ca-bundle.crt') -registry_custom_ca_bundle_storage_input_path = Path('/input/common/custom-ca-bundle.crt') \ No newline at end of file +core_cert_dir = os.path.join(config_dir, "core", "certificates") \ No newline at end of file diff --git a/make/photon/prepare/main.py b/make/photon/prepare/main.py index 8ba6c76cd..604d2735c 100644 --- a/make/photon/prepare/main.py +++ b/make/photon/prepare/main.py @@ -4,7 +4,7 @@ import click from utils.misc import delfile from utils.configs import validate, parse_yaml_config -from utils.cert import prepare_ca, SSL_CERT_KEY_PATH, SSL_CERT_PATH, get_secret_key, copy_ssl_cert, copy_secret_keys +from utils.cert import prepare_ca, SSL_CERT_KEY_PATH, SSL_CERT_PATH, get_secret_key from utils.db import prepare_db from utils.jobservice import prepare_job_service from utils.registry import prepare_registry @@ -16,13 +16,12 @@ from utils.clair import prepare_clair from utils.chart import prepare_chartmuseum from utils.docker_compose import prepare_docker_compose from utils.nginx import prepare_nginx, nginx_confd_dir -from g import (config_dir, config_file_path, private_key_pem_path, root_crt_path, -registry_custom_ca_bundle_storage_path, registry_custom_ca_bundle_storage_input_path, secret_key_dir, +from g import (config_dir, input_config_path, private_key_pem_path, root_crt_path, secret_key_dir, old_private_key_pem_path, old_crt_path) # Main function @click.command() -@click.option('--conf', default=config_file_path, help="the path of Harbor configuration file") +@click.option('--conf', default=input_config_path, help="the path of Harbor configuration file") @click.option('--with-notary', is_flag=True, help="the Harbor instance is to be deployed with notary") @click.option('--with-clair', is_flag=True, help="the Harbor instance is to be deployed with clair") @click.option('--with-chartmuseum', is_flag=True, help="the Harbor instance is to be deployed with chart repository supporting") @@ -40,21 +39,14 @@ def main(conf, with_notary, with_clair, with_chartmuseum): prepare_db(config_dict) prepare_job_service(config_dict) - copy_secret_keys() get_secret_key(secret_key_dir) - if config_dict['protocol'] == 'https': - copy_ssl_cert() - # If Customized cert enabled prepare_ca( private_key_pem_path=private_key_pem_path, root_crt_path=root_crt_path, old_private_key_pem_path=old_private_key_pem_path, - old_crt_path=old_crt_path, - registry_custom_ca_bundle_config=registry_custom_ca_bundle_storage_input_path, - registry_custom_ca_bundle_storage_path=registry_custom_ca_bundle_storage_path) - + old_crt_path=old_crt_path) if with_notary: prepare_notary(config_dict, nginx_confd_dir, SSL_CERT_PATH, SSL_CERT_KEY_PATH) diff --git a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja index 37a5dafd8..b290a543c 100644 --- a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja +++ b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja @@ -33,8 +33,11 @@ services: - {{data_volume}}/registry:/storage:z - ./common/config/registry/:/etc/registry/:z - {{data_volume}}/secret/registry/root.crt:/etc/registry/root.crt:z -{%if registry_custom_ca_bundle_storage_path %} - - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z +{% if gcs_keyfile %} + - {{gcs_keyfile}}:/etc/registry/gcs.key +{% endif %} +{%if registry_custom_ca_bundle_path %} + - {{registry_custom_ca_bundle_path}}:/harbor_cust_cert/custom-ca-bundle.crt:z {% endif %} networks: - harbor @@ -247,8 +250,8 @@ services: volumes: - ./common/config/nginx:/etc/nginx:z {% if protocol == 'https' %} - - {{data_volume}}/secret/nginx/server.key:/etc/nginx/cert/server.key - - {{data_volume}}/secret/nginx/server.crt:/etc/nginx/cert/server.crt + - {{cert_key_path}}:/etc/nginx/cert/server.key:z + - {{cert_path}}:/etc/nginx/cert/server.crt:z {% endif %} networks: - harbor @@ -257,8 +260,8 @@ services: {% endif %} dns_search: . ports: - - 80:80 - - 443:443 + - {{http_port}}:80 + - {{https_port}}:443 - 4443:4443 depends_on: - postgresql @@ -337,8 +340,8 @@ services: - postgresql volumes: - ./common/config/clair/config.yaml:/etc/clair/config.yaml:z -{%if registry_custom_ca_bundle_storage_path %} - - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z +{%if registry_custom_ca_bundle_path %} + - {{registry_custom_ca_bundle_path}}:/harbor_cust_cert/custom-ca-bundle.crt:z {% endif %} logging: driver: "syslog" @@ -368,8 +371,8 @@ services: volumes: - {{data_volume}}/chart_storage:/chart_storage:z - ./common/config/chartserver:/etc/chartserver:z -{%if registry_custom_ca_bundle_storage_path %} - - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z +{%if registry_custom_ca_bundle_path %} + - {{registry_custom_ca_bundle_path}}:/harbor_cust_cert/custom-ca-bundle.crt:z {% endif %} logging: driver: "syslog" diff --git a/make/photon/prepare/templates/nginx/nginx.http.conf.jinja b/make/photon/prepare/templates/nginx/nginx.http.conf.jinja index 27ac1d7cd..0f7f5107e 100644 --- a/make/photon/prepare/templates/nginx/nginx.http.conf.jinja +++ b/make/photon/prepare/templates/nginx/nginx.http.conf.jinja @@ -28,7 +28,7 @@ http { access_log /dev/stdout timed_combined; server { - listen {{http_port}}; + listen 80; server_tokens off; # disable any limits to avoid HTTP 413 for large image uploads client_max_body_size 0; diff --git a/make/photon/prepare/templates/nginx/nginx.https.conf.jinja b/make/photon/prepare/templates/nginx/nginx.https.conf.jinja index 41b350096..1ae2a9754 100644 --- a/make/photon/prepare/templates/nginx/nginx.https.conf.jinja +++ b/make/photon/prepare/templates/nginx/nginx.https.conf.jinja @@ -31,7 +31,7 @@ http { include /etc/nginx/conf.d/*.server.conf; server { - listen {{https_port}} ssl; + listen 443 ssl; # server_name harbordomain.com; server_tokens off; # SSL diff --git a/make/photon/prepare/templates/registry/config.yml.jinja b/make/photon/prepare/templates/registry/config.yml.jinja index 9d649d565..8af33b2d3 100644 --- a/make/photon/prepare/templates/registry/config.yml.jinja +++ b/make/photon/prepare/templates/registry/config.yml.jinja @@ -1,6 +1,6 @@ version: 0.1 log: - level: info + level: {{log_level}} fields: service: registry storage: diff --git a/make/photon/prepare/utils/cert.py b/make/photon/prepare/utils/cert.py index 260a2b574..0c75f615c 100644 --- a/make/photon/prepare/utils/cert.py +++ b/make/photon/prepare/utils/cert.py @@ -10,16 +10,11 @@ from .misc import generate_random_string SSL_CERT_PATH = os.path.join("/etc/nginx/cert", "server.crt") SSL_CERT_KEY_PATH = os.path.join("/etc/nginx/cert", "server.key") -input_cert = '/input/nginx/server.crt' -input_cert_key = '/input/nginx/server.key' - secret_cert_dir = '/secret/nginx' secret_cert = '/secret/nginx/server.crt' secret_cert_key = '/secret/nginx/server.key' -input_secret_keys_dir = '/input/keys' secret_keys_dir = '/secret/keys' -allowed_secret_key_names = ['defaultalias', 'secretkey'] def _get_secret(folder, filename, length=16): key_file = os.path.join(folder, filename) @@ -50,26 +45,6 @@ def get_alias(path): alias = _get_secret(path, "defaultalias", length=8) return alias -def copy_secret_keys(): - """ - Copy the secret keys, which used for encrypt user password, from input keys dir to secret keys dir - """ - if os.path.isdir(input_secret_keys_dir) and os.path.isdir(secret_keys_dir): - input_files = os.listdir(input_secret_keys_dir) - secret_files = os.listdir(secret_keys_dir) - files_need_copy = [x for x in input_files if (x in allowed_secret_key_names) and (x not in secret_files) ] - for f in files_need_copy: - shutil.copy(f, secret_keys_dir) - -def copy_ssl_cert(): - """ - Copy the ssl certs key paris, which used in nginx ssl certificate, from input dir to secret cert dir - """ - if os.path.isfile(input_cert_key) and os.path.isfile(input_cert): - os.makedirs(secret_cert_dir, exist_ok=True) - shutil.copy(input_cert, secret_cert) - shutil.copy(input_cert_key, secret_cert_key) - ## decorator actions def stat_decorator(func): @wraps(func) @@ -115,9 +90,7 @@ def prepare_ca( private_key_pem_path: Path, root_crt_path: Path, old_private_key_pem_path: Path, - old_crt_path: Path, - registry_custom_ca_bundle_config: Path, - registry_custom_ca_bundle_storage_path: Path): + old_crt_path: Path): if not ( private_key_pem_path.exists() and root_crt_path.exists() ): # From version 1.8 the cert storage path is changed # if old key paris not exist create new ones @@ -132,11 +105,4 @@ def prepare_ca( mark_file(root_crt_path) else: shutil.move(old_crt_path, root_crt_path) - shutil.move(old_private_key_pem_path, private_key_pem_path) - - - if not registry_custom_ca_bundle_storage_path.exists() and registry_custom_ca_bundle_config.exists(): - registry_custom_ca_bundle_storage_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(registry_custom_ca_bundle_config, registry_custom_ca_bundle_storage_path) - mark_file(registry_custom_ca_bundle_storage_path) - print("Copied custom ca bundle: %s" % registry_custom_ca_bundle_config) \ No newline at end of file + shutil.move(old_private_key_pem_path, private_key_pem_path) \ No newline at end of file diff --git a/make/photon/prepare/utils/chart.py b/make/photon/prepare/utils/chart.py index 825c80feb..68bbae87e 100644 --- a/make/photon/prepare/utils/chart.py +++ b/make/photon/prepare/utils/chart.py @@ -24,11 +24,6 @@ def prepare_chartmuseum(config_dict): print ("Create config folder: %s" % chartm_config_dir) os.makedirs(chartm_config_dir) - # handle custom ca bundle - if len(registry_custom_ca_bundle_path) > 0 and os.path.isfile(registry_custom_ca_bundle_path): - shutil.copyfile(registry_custom_ca_bundle_path, os.path.join(chartm_config_dir, "custom-ca-bundle.crt")) - print("Copied custom ca bundle: %s" % os.path.join(chartm_config_dir, "custom-ca-bundle.crt")) - # process redis info cache_store = "redis" cache_redis_password = redis_password @@ -42,18 +37,9 @@ def prepare_chartmuseum(config_dict): # storage provider configurations # please be aware that, we do not check the validations of the values for the specified keys # convert the configs to config map - storage_provider_configs = storage_provider_config.split(",") - storgae_provider_confg_map = {} + storgae_provider_confg_map = storage_provider_config storage_provider_config_options = [] - for k_v in storage_provider_configs: - if len(k_v) > 0: - kvs = k_v.split(": ") # add space suffix to avoid existing ":" in the value - if len(kvs) == 2: - #key must not be empty - if kvs[0].strip() != "": - storgae_provider_confg_map[kvs[0].strip()] = kvs[1].strip() - if storage_provider_name == "s3": # aws s3 storage storage_driver = "amazon" diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index f59348fdb..1a3b56714 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -66,24 +66,20 @@ def parse_yaml_config(config_file_path): 'chart_repository_url': 'http://chartmuseum:9999' } - config_dict['hostname'] = configs.get("hostname") - http_config = configs.get('http') - https_config = configs.get('https') + config_dict['hostname'] = configs["hostname"] + config_dict['protocol'] = 'http' + http_config = configs.get('http') or {} + config_dict['http_port'] = http_config.get('port', 80) + + https_config = configs.get('https') if https_config: config_dict['protocol'] = 'https' config_dict['https_port'] = https_config.get('port', 443) - config_dict['cert_path'] = https_config.get("certificate") - config_dict['cert_key_path'] = https_config.get("private_key") - else: - config_dict['protocol'] = 'http' - config_dict['http_port'] = http_config.get('port', 80) - - if configs.get('external_url'): - config_dict['public_url'] = configs['external_url'] - else: - config_dict['public_url'] = '{protocol}://{hostname}'.format(**config_dict) + config_dict['cert_path'] = https_config["certificate"] + config_dict['cert_key_path'] = https_config["private_key"] + config_dict['public_url'] = configs.get('external_url') or '{protocol}://{hostname}'.format(**config_dict) # DB configs db_configs = configs.get('database') @@ -91,42 +87,42 @@ def parse_yaml_config(config_file_path): config_dict['db_host'] = 'postgresql' config_dict['db_port'] = 5432 config_dict['db_user'] = 'postgres' - config_dict['db_password'] = db_configs.get("password") or 'root123' + config_dict['db_password'] = db_configs.get("password") or '' config_dict['ssl_mode'] = 'disable' # Data path volume - config_dict['data_volume'] = configs.get('data_volume') + config_dict['data_volume'] = configs['data_volume'] # Initial Admin Password - config_dict['harbor_admin_password'] = configs.get("harbor_admin_password") + config_dict['harbor_admin_password'] = configs["harbor_admin_password"] # Registry storage configs storage_config = configs.get('storage_service') or {} - if configs.get('filesystem'): - print('handle filesystem') - elif configs.get('azure'): - print('handle azure') - elif configs.get('gcs'): - print('handle gcs') - elif configs.get('s3'): - print('handle s3') - elif configs.get('swift'): - print('handle swift') - elif configs.get('oss'): - print('handle oss') + + config_dict['registry_custom_ca_bundle_path'] = storage_config.get('ca_bundle') or '' + + if storage_config.get('filesystem'): + config_dict['storage_provider_name'] = 'filesystem' + config_dict['storage_provider_config'] = storage_config['filesystem'] + elif storage_config.get('azure'): + config_dict['storage_provider_name'] = 'azure' + config_dict['storage_provider_config'] = storage_config['azure'] + elif storage_config.get('gcs'): + config_dict['storage_provider_name'] = 'gcs' + config_dict['storage_provider_config'] = storage_config['gcs'] + elif storage_config.get('s3'): + config_dict['storage_provider_name'] = 's3' + config_dict['storage_provider_config'] = storage_config['s3'] + elif storage_config.get('swift'): + config_dict['storage_provider_name'] = 'swift' + config_dict['storage_provider_config'] = storage_config['swift'] + elif storage_config.get('oss'): + config_dict['storage_provider_name'] = 'oss' + config_dict['storage_provider_config'] = storage_config['oss'] else: config_dict['storage_provider_name'] = 'filesystem' - config_dict['storage_provider_config'] = '' - config_dict['registry_custom_ca_bundle_path'] = storage_config.get("ca_bundle") or '' - - - # config_dict['storage_provider_name'] = storage_config.get("registry_storage_provider_name") or '' - # config_dict['storage_provider_config'] = storage_config.get("registry_storage_provider_config") or '' - # # yaml requires 1 or more spaces between the key and value - # config_dict['storage_provider_config'] = config_dict['storage_provider_config'].replace(":", ": ", 1) - # config_dict['registry_custom_ca_bundle_path'] = storage_config.get("registry_custom_ca_bundle") or '' - + config_dict['storage_provider_config'] = {} # Clair configs clair_configs = configs.get("clair") or {} @@ -134,45 +130,51 @@ def parse_yaml_config(config_file_path): config_dict['clair_updaters_interval'] = clair_configs.get("updaters_interval") or 12 config_dict['clair_http_proxy'] = clair_configs.get('http_proxy') or '' config_dict['clair_https_proxy'] = clair_configs.get('https_proxy') or '' - config_dict['clair_no_proxy'] = clair_configs.get('no_proxy') or '' - + config_dict['clair_no_proxy'] = clair_configs.get('no_proxy') or '127.0.0.1,localhost,core,registry' # jobservice config - js_config = configs.get('jobservice', {}) - config_dict['max_job_workers'] = js_config.get("max_job_workers", 10) + js_config = configs.get('jobservice') or {} + config_dict['max_job_workers'] = js_config["max_job_workers"] config_dict['jobservice_secret'] = generate_random_string(16) # Log configs log_configs = configs.get('log') or {} - config_dict['log_location'] = log_configs.get("location") - config_dict['log_rotate_count'] = log_configs.get("rotate_count") - config_dict['log_rotate_size'] = log_configs.get("rotate_size") - config_dict['log_level'] = log_configs.get('level') + config_dict['log_location'] = log_configs["location"] + config_dict['log_rotate_count'] = log_configs["rotate_count"] + config_dict['log_rotate_size'] = log_configs["rotate_size"] + config_dict['log_level'] = log_configs['level'] # external DB, if external_db enabled, it will cover the database config - external_db_configs = configs.get('external_database') + external_db_configs = configs.get('external_database') or {} if external_db_configs: - config_dict['db_password'] = external_db_configs.get('password') or 'root123' - if external_db_configs.get('host'): - config_dict['db_host'] = external_db_configs['host'] - if external_db_configs.get('port'): - config_dict['db_port'] = external_db_configs['port'] - if external_db_configs.get('username'): - config_dict['db_user'] = db_configs['username'] + config_dict['db_password'] = external_db_configs.get('password') or '' + config_dict['db_host'] = external_db_configs['host'] + config_dict['db_port'] = external_db_configs['port'] + config_dict['db_user'] = db_configs['username'] if external_db_configs.get('ssl_mode'): config_dict['db_ssl_mode'] = external_db_configs['ssl_mode'] - # external_redis configs - redis_configs = configs.get("external_redis") or {} - config_dict['redis_host'] = redis_configs.get("host") or 'redis' - config_dict['redis_port'] = redis_configs.get("port") or 6379 - config_dict['redis_password'] = redis_configs.get("password") or '' - config_dict['redis_db_index_reg'] = redis_configs.get('registry_db_index') or 1 - config_dict['redis_db_index_js'] = redis_configs.get('jobservice_db_index') or 2 - config_dict['redis_db_index_chart'] = redis_configs.get('chartmuseum_db_index') or 3 + # redis config + redis_configs = configs.get("external_redis") + if redis_configs: + # using external_redis + config_dict['redis_host'] = redis_configs['host'] + config_dict['redis_port'] = redis_configs['port'] + config_dict['redis_password'] = redis_configs.get("password") or '' + config_dict['redis_db_index_reg'] = redis_configs.get('registry_db_index') or 1 + config_dict['redis_db_index_js'] = redis_configs.get('jobservice_db_index') or 2 + config_dict['redis_db_index_chart'] = redis_configs.get('chartmuseum_db_index') or 3 + else: + ## Using local redis + config_dict['redis_host'] = 'redis' + config_dict['redis_port'] = 6379 + config_dict['redis_password'] = '' + config_dict['redis_db_index_reg'] = 1 + config_dict['redis_db_index_js'] = 2 + config_dict['redis_db_index_chart'] = 3 # redis://[arbitrary_username:password@]ipaddress:port/database_index if config_dict.get('redis_password'): @@ -182,14 +184,10 @@ def parse_yaml_config(config_file_path): config_dict['redis_url_js'] = "redis://%s:%s/%s" % (config_dict['redis_host'], config_dict['redis_port'], config_dict['redis_db_index_js']) config_dict['redis_url_reg'] = "redis://%s:%s/%s" % (config_dict['redis_host'], config_dict['redis_port'], config_dict['redis_db_index_reg']) - # auto generated secret string for core config_dict['core_secret'] = generate_random_string(16) # Admiral configs - if configs.get("admiral_url"): - config_dict['admiral_url'] = configs["admiral_url"] - else: - config_dict['admiral_url'] = "" + config_dict['admiral_url'] = configs.get("admiral_url") or "" return config_dict \ No newline at end of file diff --git a/make/photon/prepare/utils/docker_compose.py b/make/photon/prepare/utils/docker_compose.py index 28ece71cd..8027e36c2 100644 --- a/make/photon/prepare/utils/docker_compose.py +++ b/make/photon/prepare/utils/docker_compose.py @@ -28,10 +28,17 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum): 'cert_key_path': configs['cert_key_path'], 'cert_path': configs['cert_path'], 'protocol': configs['protocol'], - 'registry_custom_ca_bundle_storage_path': configs['registry_custom_ca_bundle_path'], + 'http_port': configs['http_port'], + 'registry_custom_ca_bundle_path': configs['registry_custom_ca_bundle_path'], 'with_notary': with_notary, 'with_clair': with_clair, 'with_chartmuseum': with_chartmuseum } + storage_config = configs.get('storage_provider_config') or {} + if storage_config.get('keyfile'): + rendering_variables['gcs_keyfile'] = storage_config['keyfile'] + if configs.get('https_port'): + rendering_variables['https_port'] = configs['https_port'] + render_jinja(docker_compose_template_path, docker_compose_yml_path, **rendering_variables) \ No newline at end of file diff --git a/make/photon/prepare/utils/nginx.py b/make/photon/prepare/utils/nginx.py index 8392be6ee..08903263a 100644 --- a/make/photon/prepare/utils/nginx.py +++ b/make/photon/prepare/utils/nginx.py @@ -23,14 +23,12 @@ def render_nginx_template(config_dict): if config_dict['protocol'] == "https": render_jinja(nginx_https_conf_template, nginx_conf, ssl_cert=SSL_CERT_PATH, - ssl_cert_key=SSL_CERT_KEY_PATH, - https_port=config_dict['https_port']) + ssl_cert_key=SSL_CERT_KEY_PATH) location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTPS else: render_jinja( nginx_http_conf_template, - nginx_conf, - http_port=config_dict['http_port']) + nginx_conf) location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTP copy_nginx_location_configs_if_exist(nginx_template_ext_dir, nginx_confd_dir, location_file_pattern) diff --git a/make/photon/prepare/utils/registry.py b/make/photon/prepare/utils/registry.py index a319465c9..c8a32463a 100644 --- a/make/photon/prepare/utils/registry.py +++ b/make/photon/prepare/utils/registry.py @@ -1,4 +1,4 @@ -import os, shutil +import os, copy from g import config_dir, templates_dir, DEFAULT_GID, DEFAULT_UID from utils.misc import prepare_config_dir @@ -11,12 +11,11 @@ registry_conf = os.path.join(config_dir, "registry", "config.yml") def prepare_registry(config_dict): - prepare_registry_config_dir() + prepare_config_dir(registry_config_dir) storage_provider_info = get_storage_provider_info( config_dict['storage_provider_name'], - config_dict['storage_provider_config'], - registry_config_dir) + config_dict['storage_provider_config']) render_jinja( registry_config_template_path, @@ -26,26 +25,17 @@ def prepare_registry(config_dict): storage_provider_info=storage_provider_info, **config_dict) -def prepare_registry_config_dir(): - prepare_config_dir(registry_config_dir) -def get_storage_provider_info(provider_name, provider_config, registry_config_dir_path): +def get_storage_provider_info(provider_name, provider_config): + provider_config_copy = copy.deepcopy(provider_config) if provider_name == "filesystem": - if not provider_config: - storage_provider_config = "rootdirectory: /storage" - elif "rootdirectory:" not in storage_provider_config: - storage_provider_config = "rootdirectory: /storage" + "," + storage_provider_config + if not (provider_config_copy and provider_config_copy.has_key('rootdirectory')): + provider_config_copy['rootdirectory'] = '/storage' + if provider_name == 'gcs' and provider_config_copy.get('keyfile'): + provider_config_copy['keyfile'] = '/etc/registry/gcs.key' # generate storage configuration section in yaml format storage_provider_conf_list = [provider_name + ':'] - for c in storage_provider_config.split(","): - kvs = c.split(": ") - if len(kvs) == 2: - if kvs[0].strip() == "keyfile": - srcKeyFile = kvs[1].strip() - if os.path.isfile(srcKeyFile): - shutil.copyfile(srcKeyFile, os.path.join(registry_config_dir_path, "gcs.key")) - storage_provider_conf_list.append("keyfile: %s" % "/etc/registry/gcs.key") - continue - storage_provider_conf_list.append(c.strip()) + for config in provider_config_copy.items(): + storage_provider_conf_list.append('{}: {}'.format(*config)) storage_provider_info = ('\n' + ' ' * 4).join(storage_provider_conf_list) return storage_provider_info diff --git a/make/prepare b/make/prepare index 8907087d6..a2871b659 100755 --- a/make/prepare +++ b/make/prepare @@ -1,35 +1,40 @@ #!/bin/bash +set +e # If compling source code this dir is harbor's make dir # If install harbor via pacakge, this dir is harbor's root dir harbor_prepare_path="$( cd "$(dirname "$0")" ; pwd -P )" - -echo host make path is set to ${harbor_prepare_path} +echo "prepare base dir is set to ${harbor_prepare_path}" data_path=$(grep '^[^#]*data_volume:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') -log_path=$(grep '^[^#]*location:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') -ssl_cert_path=$(grep '^[^#]*ssl_cert:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') -ssl_cert_key_path=$(grep '^[^#]*ssl_cert_key:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') -registry_custom_ca_bundle=$(grep '^[^#]*registry_custom_ca_bundle:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') +# If previous secretkeys exist, move it to new location +previous_secretkey_path=/data/secretkey +previous_defaultalias_path=/data/defaultalias + +if [ -f $previous_secretkey_path ]; then + mkdir -p $data_path/secret/keys + mv $previous_secretkey_path $data_path/secret/keys +fi +if [ -f $previous_defaultalias_path ]; then + mkdir -p $data_path/secret/keys + mv $previous_defaultalias_path $data_path/secret/keys +fi + +# Clean up input dir +rm -rf ${harbor_prepare_path}/input # Create a input dirs mkdir -p ${harbor_prepare_path}/input input_dir=${harbor_prepare_path}/input -mkdir -p $input_dir/nginx -mkdir -p $input_dir/keys -mkdir -p $input_dir/common -# Copy nginx config file to input dir -cp $ssl_cert_path $input_dir/nginx/server.crt -cp $ssl_cert_key_path $input_dir/nginx/server.key - -# Copy ca bundle to input dir -if [ -f $registry_custom_ca_bundle ] -then - cp -r $registry_custom_ca_bundle $input_dir/common/custom-ca-bundle.crt -fi +set -e # Copy harbor.yml to input dir -cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml +if [[ ! "$1" =~ ^\-\- ]] && [ -f "$1" ] +then + cp $1 $input_dir/harbor.yml +else + cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml +fi # Create secret dir secret_dir=${data_path}/secret @@ -40,8 +45,8 @@ docker run -it --rm -v $input_dir:/input \ -v $harbor_prepare_path:/compose_location \ -v $config_dir:/config \ -v $secret_dir:/secret \ - -v $log_path:/var/log/harbor \ goharbor/prepare:dev $@ +echo "Clean up the input dir" # Clean up input dir -rm -rf ${harbor_prepare_path}/input \ No newline at end of file +rm -rf ${harbor_prepare_path}/input diff --git a/tests/hostcfg.sh b/tests/hostcfg.sh index e1109fe13..9aae9c3a0 100755 --- a/tests/hostcfg.sh +++ b/tests/hostcfg.sh @@ -1,7 +1,9 @@ #!/bin/bash IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'` -PROTOCOL='https' #echo $IP sudo sed "s/reg.mydomain.com/$IP/" -i make/harbor.yml -sudo sed "s/^ui_url_protocol: .*/ui_url_protocol: $PROTOCOL/g" -i make/harbor.yml + +echo "https:" >> make/harbor.yml +echo " certificate: /data/cert/server.crt" >> make/harbor.yml +echo " private_key: /data/cert/server.key" >> make/harbor.yml \ No newline at end of file diff --git a/tests/testprepare.sh b/tests/testprepare.sh index 264270ad0..3776b597a 100755 --- a/tests/testprepare.sh +++ b/tests/testprepare.sh @@ -8,7 +8,7 @@ cp /data/secret/core/private_key.pem /etc/core/ mkdir src/core/conf cp make/common/config/core/app.conf src/core/conf/ if [ "$(uname)" == "Darwin" ]; then - IP=`ifconfig en0 | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}'` + IP=`ifconfig en0 | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}'` else IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'` fi diff --git a/tests/travis/api_common_install.sh b/tests/travis/api_common_install.sh index 3d06c8081..504157733 100644 --- a/tests/travis/api_common_install.sh +++ b/tests/travis/api_common_install.sh @@ -10,12 +10,12 @@ sudo sed "s/127.0.0.1/$1/" -i tests/generateCerts.sh sudo ./tests/generateCerts.sh sudo mkdir -p /etc/docker/certs.d/$1 && sudo cp ./harbor_ca.crt /etc/docker/certs.d/$1/ +sudo ./tests/hostcfg.sh + if [ "$2" = 'LDAP' ]; then - sudo ./tests/hostcfg.sh LDAP cd tests && sudo ./ldapprepare.sh && cd .. fi -sudo ./tests/hostcfg.sh # prepare a chart file for API_DB test...