Enhance: Refacotr Registry config file

1. Refactor registry configs
2. cp gcs keyfile is exist

Signed-off-by: Qian Deng <dengq@vmware.com>

Makefile

@@ -295,7 +295,7 @@ compile: check_environment versions_prepare compile_core compile_jobservice comp
 
 update_prepare_version:
 	@echo "substitude the prepare version tag in prepare file..."
-	$(SEDCMD) -i -e 's/goharbor\/prepare:.*[[:space:]]\+/goharbor\/prepare:$(VERSIONTAG) /' $(MAKEPATH)/prepare ;
+	@$(SEDCMD) -i -e 's/goharbor\/prepare:.*[[:space:]]\+/goharbor\/prepare:$(VERSIONTAG) /' $(MAKEPATH)/prepare ;
 
 prepare: update_prepare_version
 	@echo "preparing..."
@@ -310,7 +310,7 @@ build:
 
 install: compile ui_version build prepare start
 
-package_online: prepare
+package_online: update_prepare_version
 	@echo "packing online package ..."
 	@cp -r make $(HARBORPKG)
 	@if [ -n "$(REGISTRYSERVER)" ] ; then \

make/harbor.yml

@@ -9,7 +9,7 @@ http:
 
 # https:
 #   port: 443
-#   #The path of cert and key files for nginx, they are applied only the protocol is set to https
+#   #The path of cert and key files for nginx
 #   certificate: /your/certificate/path
 #   private_key: /your/private/key/path
 
@@ -30,64 +30,19 @@ database:
 # The default data volume
 data_volume: /data
 
-# Harbor Storage settings
+# Harbor Storage settings by default is using /data dir on local filesystem
-storage_service:
+# Uncomment storage_service setting If you want to using external storage
-  # registry_custom_ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
+# storage_service:
-  # of registry's and chart repository's containers.  This is usually needed when the user hosts a internal storage with self signed certificate.
+#   # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
-  ca_bundle:
+#   # of registry's and chart repository's containers.  This is usually needed when the user hosts a internal storage with self signed certificate.
+#   ca_bundle:
 
-  # storaged in $data_volume/
+#   # storage backend, default is filesystem, options include filesystem, azure, gcs, s3, swift and oss
-  # filesystem: 
+#   # for more info about this configuration please refer https://docs.docker.com/registry/configuration/
-  #   maxthreads: 100
+#   filesystem:
-  # azure:
+#     maxthreads: 100
-  #   accountname: accountname
-  #   accountkey: base64encodedaccountkey
-  #   container: containername
-  # gcs:
-  #   bucket: bucketname
-  #   keyfile: /path/to/keyfile
-  #   rootdirectory: /gcs/object/name/prefix
-  #   chunksize: 5242880
-  # s3:
-  #   accesskey: awsaccesskey
-  #   secretkey: awssecretkey
-  #   region: us-west-1
-  #   regionendpoint: http://myobjects.local
-  #   bucket: bucketname
-  #   encrypt: true
-  #   keyid: mykeyid
-  #   secure: true
-  #   v4auth: true
-  #   chunksize: 5242880
-  #   multipartcopychunksize: 33554432
-  #   multipartcopymaxconcurrency: 100
-  #   multipartcopythresholdsize: 33554432
-  #   rootdirectory: /s3/object/name/prefix
-  # swift:
-  #   username: username
-  #   password: password
-  #   authurl: https://storage.myprovider.com/auth/v1.0 or https://storage.myprovider.com/v2.0 or https://storage.myprovider.com/v3/auth
-  #   tenant: tenantname
-  #   tenantid: tenantid
-  #   domain: domain name for Openstack Identity v3 API
-  #   domainid: domain id for Openstack Identity v3 API
-  #   insecureskipverify: true
-  #   region: fr
-  #   container: containername
-  #   rootdirectory: /swift/object/name/prefix
-  # oss:
-  #   accesskeyid: accesskeyid
-  #   accesskeysecret: accesskeysecret
-  #   region: OSS region name
-  #   endpoint: optional endpoints
-  #   internal: optional internal endpoint
-  #   bucket: OSS bucket
-  #   encrypt: optional data encryption setting
-  #   secure: optional ssl setting
-  #   chunksize: optional size valye
-  #   rootdirectory: optional root directory
 
-# Clair DB configuration
+# Clair configuration
 clair: 
   # The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
   updaters_interval: 12
@@ -118,7 +73,8 @@ log:
 #This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
 _version: 1.7.0
 
-# Uncomment if using external database. Only support postgres.
+# Uncomment external_database if using external database. And the password will replace the the password setting in database.
+# And currently ontly support postgres.
 # external_database:
 #   host: postgresql
 #   port: 5432
@@ -126,7 +82,7 @@ _version: 1.7.0
 #   password: root123
 #   ssl_mode: disable
 
-# Umcomments if using external Redis server
+# Umcomments external_redis if using external Redis server
 # external_redis:
 #   host: redis
 #   port: 6379
@@ -134,4 +90,4 @@ _version: 1.7.0
 #   # db_index 0 is for core, it's unchangeable
 #   registry_db_index: 1
 #   jobservice_db_index: 2
-#   chartmuseum_db_index: 3
+#   chartmuseum_db_index: 3

make/install.sh

@@ -192,9 +192,9 @@ docker-compose up -d
 protocol=http
 hostname=reg.mydomain.com
 
-if [[ $(cat ./harbor.yml) =~ ui_url_protocol:[[:blank:]]*(https?) ]]
+if [ -n "$(grep '^[^#]*https:' ./harbor.yml)"]
 then
-protocol=${BASH_REMATCH[1]}
+protocol=https
 fi
 
 if [[ $(grep '^[[:blank:]]*hostname:' ./harbor.yml) =~ hostname:[[:blank:]]*(.*) ]]

make/photon/prepare/g.py

@@ -20,10 +20,8 @@ private_key_pem_path = Path('/secret/core/private_key.pem')
 root_crt_path = Path('/secret/registry/root.crt')
 
 config_file_path = '/compose_location/harbor.yml'
+input_config_path = '/input/harbor.yml'
 versions_file_path = Path('/usr/src/app/versions')
 
 cert_dir = os.path.join(config_dir, "nginx", "cert")
-core_cert_dir = os.path.join(config_dir, "core", "certificates")
+core_cert_dir = os.path.join(config_dir, "core", "certificates")
-
-registry_custom_ca_bundle_storage_path = Path('/secret/common/custom-ca-bundle.crt')
-registry_custom_ca_bundle_storage_input_path = Path('/input/common/custom-ca-bundle.crt')

make/photon/prepare/main.py

@@ -4,7 +4,7 @@ import click
 
 from utils.misc import delfile
 from utils.configs import validate, parse_yaml_config
-from utils.cert import prepare_ca, SSL_CERT_KEY_PATH, SSL_CERT_PATH, get_secret_key, copy_ssl_cert, copy_secret_keys
+from utils.cert import prepare_ca, SSL_CERT_KEY_PATH, SSL_CERT_PATH, get_secret_key
 from utils.db import prepare_db
 from utils.jobservice import prepare_job_service
 from utils.registry import prepare_registry
@@ -16,13 +16,12 @@ from utils.clair import prepare_clair
 from utils.chart import prepare_chartmuseum
 from utils.docker_compose import prepare_docker_compose
 from utils.nginx import prepare_nginx, nginx_confd_dir
-from g import (config_dir, config_file_path, private_key_pem_path, root_crt_path,
+from g import (config_dir, input_config_path, private_key_pem_path, root_crt_path, secret_key_dir,
-registry_custom_ca_bundle_storage_path, registry_custom_ca_bundle_storage_input_path, secret_key_dir,
 old_private_key_pem_path, old_crt_path)
 
 # Main function
 @click.command()
-@click.option('--conf', default=config_file_path, help="the path of Harbor configuration file")
+@click.option('--conf', default=input_config_path, help="the path of Harbor configuration file")
 @click.option('--with-notary', is_flag=True, help="the Harbor instance is to be deployed with notary")
 @click.option('--with-clair', is_flag=True, help="the Harbor instance is to be deployed with clair")
 @click.option('--with-chartmuseum', is_flag=True, help="the Harbor instance is to be deployed with chart repository supporting")
@@ -40,21 +39,14 @@ def main(conf, with_notary, with_clair, with_chartmuseum):
     prepare_db(config_dict)
     prepare_job_service(config_dict)
 
-    copy_secret_keys()
     get_secret_key(secret_key_dir)
 
-    if config_dict['protocol'] == 'https':
-        copy_ssl_cert()
-
     #  If Customized cert enabled
     prepare_ca(
         private_key_pem_path=private_key_pem_path,
         root_crt_path=root_crt_path,
         old_private_key_pem_path=old_private_key_pem_path,
-        old_crt_path=old_crt_path,
+        old_crt_path=old_crt_path)
-        registry_custom_ca_bundle_config=registry_custom_ca_bundle_storage_input_path,
-        registry_custom_ca_bundle_storage_path=registry_custom_ca_bundle_storage_path)
-
     if with_notary:
         prepare_notary(config_dict, nginx_confd_dir, SSL_CERT_PATH, SSL_CERT_KEY_PATH)
 

make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja

@@ -33,8 +33,11 @@ services:
       - {{data_volume}}/registry:/storage:z
       - ./common/config/registry/:/etc/registry/:z
       - {{data_volume}}/secret/registry/root.crt:/etc/registry/root.crt:z
-{%if registry_custom_ca_bundle_storage_path %}
+{% if gcs_keyfile %}
-      - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z
+      - {{gcs_keyfile}}:/etc/registry/gcs.key
+{% endif %}
+{%if registry_custom_ca_bundle_path %}
+      - {{registry_custom_ca_bundle_path}}:/harbor_cust_cert/custom-ca-bundle.crt:z
 {% endif %}
     networks:
       - harbor
@@ -247,8 +250,8 @@ services:
     volumes:
       - ./common/config/nginx:/etc/nginx:z
 {% if protocol == 'https' %}
-      - {{data_volume}}/secret/nginx/server.key:/etc/nginx/cert/server.key
+      - {{cert_key_path}}:/etc/nginx/cert/server.key:z
-      - {{data_volume}}/secret/nginx/server.crt:/etc/nginx/cert/server.crt
+      - {{cert_path}}:/etc/nginx/cert/server.crt:z
 {% endif %}
     networks:
       - harbor
@@ -257,8 +260,8 @@ services:
 {% endif %}
     dns_search: .
     ports:
-      - 80:80
+      - {{http_port}}:80
-      - 443:443
+      - {{https_port}}:443
       - 4443:4443
     depends_on:
       - postgresql
@@ -337,8 +340,8 @@ services:
       - postgresql
     volumes:
       - ./common/config/clair/config.yaml:/etc/clair/config.yaml:z
-{%if registry_custom_ca_bundle_storage_path %}
+{%if registry_custom_ca_bundle_path %}
-      - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z
+      - {{registry_custom_ca_bundle_path}}:/harbor_cust_cert/custom-ca-bundle.crt:z
 {% endif %}
     logging:
       driver: "syslog"
@@ -368,8 +371,8 @@ services:
     volumes:
       - {{data_volume}}/chart_storage:/chart_storage:z
       - ./common/config/chartserver:/etc/chartserver:z
-{%if registry_custom_ca_bundle_storage_path %}
+{%if registry_custom_ca_bundle_path %}
-      - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z
+      - {{registry_custom_ca_bundle_path}}:/harbor_cust_cert/custom-ca-bundle.crt:z
 {% endif %}
     logging:
       driver: "syslog"

make/photon/prepare/templates/nginx/nginx.http.conf.jinja

@@ -28,7 +28,7 @@ http {
   access_log /dev/stdout timed_combined;
 
   server {
-    listen {{http_port}};
+    listen 80;
     server_tokens off;
     # disable any limits to avoid HTTP 413 for large image uploads
     client_max_body_size 0;

make/photon/prepare/templates/nginx/nginx.https.conf.jinja

@@ -31,7 +31,7 @@ http {
   include /etc/nginx/conf.d/*.server.conf;
 
   server {
-    listen {{https_port}} ssl;
+    listen 443 ssl;
 #    server_name harbordomain.com;
     server_tokens off;
     # SSL

make/photon/prepare/templates/registry/config.yml.jinja

@@ -1,6 +1,6 @@
 version: 0.1
 log:
-  level: info
+  level: {{log_level}}
   fields:
     service: registry
 storage:

make/photon/prepare/utils/cert.py

@@ -10,16 +10,11 @@ from .misc import generate_random_string
 SSL_CERT_PATH = os.path.join("/etc/nginx/cert", "server.crt")
 SSL_CERT_KEY_PATH = os.path.join("/etc/nginx/cert", "server.key")
 
-input_cert = '/input/nginx/server.crt'
-input_cert_key = '/input/nginx/server.key'
-
 secret_cert_dir = '/secret/nginx'
 secret_cert = '/secret/nginx/server.crt'
 secret_cert_key = '/secret/nginx/server.key'
 
-input_secret_keys_dir = '/input/keys'
 secret_keys_dir = '/secret/keys'
-allowed_secret_key_names = ['defaultalias', 'secretkey']
 
 def _get_secret(folder, filename, length=16):
     key_file = os.path.join(folder, filename)
@@ -50,26 +45,6 @@ def get_alias(path):
     alias = _get_secret(path, "defaultalias", length=8)
     return alias
 
-def copy_secret_keys():
-    """
-    Copy the secret keys, which used for encrypt user password, from input keys dir to secret keys dir
-    """
-    if os.path.isdir(input_secret_keys_dir) and os.path.isdir(secret_keys_dir):
-        input_files = os.listdir(input_secret_keys_dir)
-        secret_files = os.listdir(secret_keys_dir)
-        files_need_copy =  [x for x in input_files if (x in allowed_secret_key_names) and (x not in secret_files) ]
-        for f in files_need_copy:
-            shutil.copy(f, secret_keys_dir)
-
-def copy_ssl_cert():
-    """
-    Copy the ssl certs key paris, which used in nginx ssl certificate, from input dir to secret cert dir
-    """
-    if os.path.isfile(input_cert_key) and os.path.isfile(input_cert):
-        os.makedirs(secret_cert_dir, exist_ok=True)
-        shutil.copy(input_cert, secret_cert)
-        shutil.copy(input_cert_key, secret_cert_key)
-
 ## decorator actions
 def stat_decorator(func):
     @wraps(func)
@@ -115,9 +90,7 @@ def prepare_ca(
     private_key_pem_path: Path,
     root_crt_path: Path,
     old_private_key_pem_path: Path,
-    old_crt_path: Path,
+    old_crt_path: Path):
-    registry_custom_ca_bundle_config: Path,
-    registry_custom_ca_bundle_storage_path: Path):
     if not ( private_key_pem_path.exists() and root_crt_path.exists() ):
         # From version 1.8 the cert storage path is changed
         # if old key paris not exist create new ones
@@ -132,11 +105,4 @@ def prepare_ca(
             mark_file(root_crt_path)
         else:
             shutil.move(old_crt_path, root_crt_path)
-            shutil.move(old_private_key_pem_path, private_key_pem_path)
+            shutil.move(old_private_key_pem_path, private_key_pem_path)
-
-
-    if not registry_custom_ca_bundle_storage_path.exists() and registry_custom_ca_bundle_config.exists():
-        registry_custom_ca_bundle_storage_path.parent.mkdir(parents=True, exist_ok=True)
-        shutil.copyfile(registry_custom_ca_bundle_config, registry_custom_ca_bundle_storage_path)
-        mark_file(registry_custom_ca_bundle_storage_path)
-        print("Copied custom ca bundle: %s" % registry_custom_ca_bundle_config)

make/photon/prepare/utils/chart.py

@@ -24,11 +24,6 @@ def prepare_chartmuseum(config_dict):
         print ("Create config folder: %s" % chartm_config_dir)
         os.makedirs(chartm_config_dir)
 
-    # handle custom ca bundle
-    if len(registry_custom_ca_bundle_path) > 0 and os.path.isfile(registry_custom_ca_bundle_path):
-        shutil.copyfile(registry_custom_ca_bundle_path, os.path.join(chartm_config_dir, "custom-ca-bundle.crt"))
-        print("Copied custom ca bundle: %s" % os.path.join(chartm_config_dir, "custom-ca-bundle.crt"))
-
     # process redis info
     cache_store = "redis"
     cache_redis_password = redis_password
@@ -42,18 +37,9 @@ def prepare_chartmuseum(config_dict):
     # storage provider configurations
     # please be aware that, we do not check the validations of the values for the specified keys
     # convert the configs to config map
-    storage_provider_configs = storage_provider_config.split(",")
+    storgae_provider_confg_map = storage_provider_config
-    storgae_provider_confg_map = {}
     storage_provider_config_options = []
 
-    for k_v in storage_provider_configs:
-        if len(k_v) > 0:
-            kvs = k_v.split(": ") # add space suffix to avoid existing ":" in the value
-            if len(kvs) == 2:
-                #key must not be empty
-                if kvs[0].strip() != "":
-                    storgae_provider_confg_map[kvs[0].strip()] = kvs[1].strip()
-
     if storage_provider_name == "s3":
         # aws s3 storage
         storage_driver = "amazon"

make/photon/prepare/utils/configs.py

@@ -66,24 +66,20 @@ def parse_yaml_config(config_file_path):
         'chart_repository_url': 'http://chartmuseum:9999'
     }
 
-    config_dict['hostname'] = configs.get("hostname")
+    config_dict['hostname'] = configs["hostname"]
-    http_config = configs.get('http')
-    https_config = configs.get('https')
 
+    config_dict['protocol'] = 'http'
+    http_config = configs.get('http') or {}
+    config_dict['http_port'] = http_config.get('port', 80)
+
+    https_config = configs.get('https')
     if https_config:
         config_dict['protocol'] = 'https'
         config_dict['https_port'] = https_config.get('port', 443)
-        config_dict['cert_path'] = https_config.get("certificate")
+        config_dict['cert_path'] = https_config["certificate"]
-        config_dict['cert_key_path'] = https_config.get("private_key")
+        config_dict['cert_key_path'] = https_config["private_key"]
-    else:
-        config_dict['protocol'] = 'http'
-        config_dict['http_port'] = http_config.get('port', 80)
-
-    if configs.get('external_url'):
-        config_dict['public_url'] = configs['external_url']
-    else:
-        config_dict['public_url'] = '{protocol}://{hostname}'.format(**config_dict)
 
+    config_dict['public_url'] = configs.get('external_url') or '{protocol}://{hostname}'.format(**config_dict)
 
     # DB configs
     db_configs = configs.get('database')
@@ -91,42 +87,42 @@ def parse_yaml_config(config_file_path):
         config_dict['db_host'] = 'postgresql'
         config_dict['db_port'] = 5432
         config_dict['db_user'] = 'postgres'
-        config_dict['db_password'] = db_configs.get("password") or 'root123'
+        config_dict['db_password'] = db_configs.get("password") or ''
         config_dict['ssl_mode'] = 'disable'
 
 
     # Data path volume
-    config_dict['data_volume'] = configs.get('data_volume')
+    config_dict['data_volume'] = configs['data_volume']
 
     # Initial Admin Password
-    config_dict['harbor_admin_password'] = configs.get("harbor_admin_password")
+    config_dict['harbor_admin_password'] = configs["harbor_admin_password"]
 
     # Registry storage configs
     storage_config = configs.get('storage_service') or {}
-    if configs.get('filesystem'):
+
-        print('handle filesystem')
+    config_dict['registry_custom_ca_bundle_path'] = storage_config.get('ca_bundle') or ''
-    elif configs.get('azure'):
+
-        print('handle azure')
+    if storage_config.get('filesystem'):
-    elif configs.get('gcs'):
+        config_dict['storage_provider_name'] = 'filesystem'
-        print('handle gcs')
+        config_dict['storage_provider_config'] = storage_config['filesystem']
-    elif configs.get('s3'):
+    elif storage_config.get('azure'):
-        print('handle s3')
+        config_dict['storage_provider_name'] = 'azure'
-    elif configs.get('swift'):
+        config_dict['storage_provider_config'] = storage_config['azure']
-        print('handle swift')
+    elif storage_config.get('gcs'):
-    elif configs.get('oss'):
+        config_dict['storage_provider_name'] = 'gcs'
-        print('handle oss')
+        config_dict['storage_provider_config'] = storage_config['gcs']
+    elif storage_config.get('s3'):
+        config_dict['storage_provider_name'] = 's3'
+        config_dict['storage_provider_config'] = storage_config['s3']
+    elif storage_config.get('swift'):
+        config_dict['storage_provider_name'] = 'swift'
+        config_dict['storage_provider_config'] = storage_config['swift']
+    elif storage_config.get('oss'):
+        config_dict['storage_provider_name'] = 'oss'
+        config_dict['storage_provider_config'] = storage_config['oss']
     else:
         config_dict['storage_provider_name'] = 'filesystem'
-        config_dict['storage_provider_config'] = ''
+        config_dict['storage_provider_config'] = {}
-        config_dict['registry_custom_ca_bundle_path'] = storage_config.get("ca_bundle") or ''
-
-
-    # config_dict['storage_provider_name'] = storage_config.get("registry_storage_provider_name") or ''
-    # config_dict['storage_provider_config'] = storage_config.get("registry_storage_provider_config") or ''
-    # # yaml requires 1 or more spaces between the key and value
-    # config_dict['storage_provider_config'] = config_dict['storage_provider_config'].replace(":", ": ", 1)
-    # config_dict['registry_custom_ca_bundle_path'] = storage_config.get("registry_custom_ca_bundle") or ''
-
 
     # Clair configs
     clair_configs = configs.get("clair") or {}
@@ -134,45 +130,51 @@ def parse_yaml_config(config_file_path):
     config_dict['clair_updaters_interval'] = clair_configs.get("updaters_interval") or 12
     config_dict['clair_http_proxy'] = clair_configs.get('http_proxy') or ''
     config_dict['clair_https_proxy'] = clair_configs.get('https_proxy') or ''
-    config_dict['clair_no_proxy'] = clair_configs.get('no_proxy') or ''
+    config_dict['clair_no_proxy'] = clair_configs.get('no_proxy') or '127.0.0.1,localhost,core,registry'
-
 
     # jobservice config
-    js_config = configs.get('jobservice', {})
+    js_config = configs.get('jobservice') or {}
-    config_dict['max_job_workers'] = js_config.get("max_job_workers", 10)
+    config_dict['max_job_workers'] = js_config["max_job_workers"]
     config_dict['jobservice_secret'] = generate_random_string(16)
 
 
     # Log configs
     log_configs = configs.get('log') or {}
-    config_dict['log_location'] = log_configs.get("location")
+    config_dict['log_location'] = log_configs["location"]
-    config_dict['log_rotate_count'] = log_configs.get("rotate_count")
+    config_dict['log_rotate_count'] = log_configs["rotate_count"]
-    config_dict['log_rotate_size'] = log_configs.get("rotate_size")
+    config_dict['log_rotate_size'] = log_configs["rotate_size"]
-    config_dict['log_level'] = log_configs.get('level')
+    config_dict['log_level'] = log_configs['level']
 
 
     # external DB, if external_db enabled, it will cover the database config
-    external_db_configs = configs.get('external_database')
+    external_db_configs = configs.get('external_database') or {}
     if external_db_configs:
-        config_dict['db_password'] = external_db_configs.get('password') or 'root123'
+        config_dict['db_password'] = external_db_configs.get('password') or ''
-        if external_db_configs.get('host'):
+        config_dict['db_host'] = external_db_configs['host']
-            config_dict['db_host'] = external_db_configs['host']
+        config_dict['db_port'] = external_db_configs['port']
-        if external_db_configs.get('port'):
+        config_dict['db_user'] = db_configs['username']
-            config_dict['db_port'] = external_db_configs['port']
-        if  external_db_configs.get('username'):
-            config_dict['db_user'] = db_configs['username']
         if external_db_configs.get('ssl_mode'):
             config_dict['db_ssl_mode'] = external_db_configs['ssl_mode']
 
 
-    # external_redis configs
+    # redis config
-    redis_configs = configs.get("external_redis") or {}
+    redis_configs = configs.get("external_redis")
-    config_dict['redis_host'] = redis_configs.get("host") or 'redis'
+    if redis_configs:
-    config_dict['redis_port'] = redis_configs.get("port") or 6379
+        # using external_redis
-    config_dict['redis_password'] = redis_configs.get("password") or ''
+        config_dict['redis_host'] = redis_configs['host']
-    config_dict['redis_db_index_reg'] = redis_configs.get('registry_db_index') or 1
+        config_dict['redis_port'] = redis_configs['port']
-    config_dict['redis_db_index_js'] = redis_configs.get('jobservice_db_index') or 2
+        config_dict['redis_password'] = redis_configs.get("password") or ''
-    config_dict['redis_db_index_chart'] = redis_configs.get('chartmuseum_db_index') or 3
+        config_dict['redis_db_index_reg'] = redis_configs.get('registry_db_index') or 1
+        config_dict['redis_db_index_js'] = redis_configs.get('jobservice_db_index') or 2
+        config_dict['redis_db_index_chart'] = redis_configs.get('chartmuseum_db_index') or 3
+    else:
+        ## Using local redis
+        config_dict['redis_host'] = 'redis'
+        config_dict['redis_port'] = 6379
+        config_dict['redis_password'] = ''
+        config_dict['redis_db_index_reg'] = 1
+        config_dict['redis_db_index_js'] = 2
+        config_dict['redis_db_index_chart'] = 3
 
     # redis://[arbitrary_username:password@]ipaddress:port/database_index
     if config_dict.get('redis_password'):
@@ -182,14 +184,10 @@ def parse_yaml_config(config_file_path):
         config_dict['redis_url_js'] = "redis://%s:%s/%s" % (config_dict['redis_host'], config_dict['redis_port'], config_dict['redis_db_index_js'])
         config_dict['redis_url_reg'] = "redis://%s:%s/%s" % (config_dict['redis_host'], config_dict['redis_port'], config_dict['redis_db_index_reg'])
 
-
     # auto generated secret string for core
     config_dict['core_secret'] = generate_random_string(16)
 
      # Admiral configs
-    if configs.get("admiral_url"):
+    config_dict['admiral_url'] = configs.get("admiral_url") or ""
-        config_dict['admiral_url'] = configs["admiral_url"]
-    else:
-        config_dict['admiral_url'] = ""
 
     return config_dict

make/photon/prepare/utils/docker_compose.py

@@ -28,10 +28,17 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):
         'cert_key_path': configs['cert_key_path'],
         'cert_path': configs['cert_path'],
         'protocol': configs['protocol'],
-        'registry_custom_ca_bundle_storage_path': configs['registry_custom_ca_bundle_path'],
+        'http_port': configs['http_port'],
+        'registry_custom_ca_bundle_path': configs['registry_custom_ca_bundle_path'],
         'with_notary': with_notary,
         'with_clair': with_clair,
         'with_chartmuseum': with_chartmuseum
     }
 
+    storage_config = configs.get('storage_provider_config') or {}
+    if storage_config.get('keyfile'):
+        rendering_variables['gcs_keyfile'] = storage_config['keyfile']
+    if configs.get('https_port'):
+        rendering_variables['https_port'] = configs['https_port']
+
     render_jinja(docker_compose_template_path, docker_compose_yml_path, **rendering_variables)

make/photon/prepare/utils/nginx.py

@@ -23,14 +23,12 @@ def render_nginx_template(config_dict):
     if config_dict['protocol'] == "https":
         render_jinja(nginx_https_conf_template, nginx_conf,
             ssl_cert=SSL_CERT_PATH,
-            ssl_cert_key=SSL_CERT_KEY_PATH,
+            ssl_cert_key=SSL_CERT_KEY_PATH)
-            https_port=config_dict['https_port'])
         location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTPS
     else:
         render_jinja(
             nginx_http_conf_template,
-            nginx_conf,
+            nginx_conf)
-            http_port=config_dict['http_port'])
         location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTP
     copy_nginx_location_configs_if_exist(nginx_template_ext_dir, nginx_confd_dir, location_file_pattern)
 

make/photon/prepare/utils/registry.py

@@ -1,4 +1,4 @@
-import os, shutil
+import os, copy
 
 from g import config_dir, templates_dir, DEFAULT_GID, DEFAULT_UID
 from utils.misc import prepare_config_dir
@@ -11,12 +11,11 @@ registry_conf = os.path.join(config_dir, "registry", "config.yml")
 
 
 def prepare_registry(config_dict):
-    prepare_registry_config_dir()
+    prepare_config_dir(registry_config_dir)
 
     storage_provider_info = get_storage_provider_info(
     config_dict['storage_provider_name'],
-    config_dict['storage_provider_config'],
+    config_dict['storage_provider_config'])
-    registry_config_dir)
 
     render_jinja(
         registry_config_template_path,
@@ -26,26 +25,17 @@ def prepare_registry(config_dict):
         storage_provider_info=storage_provider_info,
         **config_dict)
 
-def prepare_registry_config_dir():
-    prepare_config_dir(registry_config_dir)
 
-def get_storage_provider_info(provider_name, provider_config, registry_config_dir_path):
+def get_storage_provider_info(provider_name, provider_config):
+    provider_config_copy = copy.deepcopy(provider_config)
     if provider_name == "filesystem":
-        if not provider_config:
+        if not (provider_config_copy and provider_config_copy.has_key('rootdirectory')):
-            storage_provider_config = "rootdirectory: /storage"
+            provider_config_copy['rootdirectory'] = '/storage'
-        elif "rootdirectory:" not in storage_provider_config:
+    if provider_name == 'gcs' and provider_config_copy.get('keyfile'):
-            storage_provider_config = "rootdirectory: /storage" + "," + storage_provider_config
+        provider_config_copy['keyfile'] = '/etc/registry/gcs.key'
     # generate storage configuration section in yaml format
     storage_provider_conf_list = [provider_name + ':']
-    for c in storage_provider_config.split(","):
+    for config in provider_config_copy.items():
-        kvs = c.split(": ")
+        storage_provider_conf_list.append('{}: {}'.format(*config))
-        if len(kvs) == 2:
-            if kvs[0].strip() == "keyfile":
-                srcKeyFile = kvs[1].strip()
-                if os.path.isfile(srcKeyFile):
-                    shutil.copyfile(srcKeyFile, os.path.join(registry_config_dir_path, "gcs.key"))
-                    storage_provider_conf_list.append("keyfile: %s" % "/etc/registry/gcs.key")
-                    continue
-        storage_provider_conf_list.append(c.strip())
     storage_provider_info = ('\n' + ' ' * 4).join(storage_provider_conf_list)
     return storage_provider_info

make/prepare

@@ -1,35 +1,40 @@
 #!/bin/bash
+set +e
 
 # If compling source code this dir is harbor's make dir
 # If install harbor via pacakge, this dir is harbor's root dir
 harbor_prepare_path="$( cd "$(dirname "$0")" ; pwd -P )"
-
+echo "prepare base dir is set to ${harbor_prepare_path}"
-echo host make path is set to ${harbor_prepare_path}
 data_path=$(grep '^[^#]*data_volume:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}')
-log_path=$(grep '^[^#]*location:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}')
-ssl_cert_path=$(grep '^[^#]*ssl_cert:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}')
-ssl_cert_key_path=$(grep '^[^#]*ssl_cert_key:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}')
-registry_custom_ca_bundle=$(grep '^[^#]*registry_custom_ca_bundle:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}')
 
+# If previous secretkeys exist, move it to new location
+previous_secretkey_path=/data/secretkey
+previous_defaultalias_path=/data/defaultalias
+
+if [ -f $previous_secretkey_path ]; then
+    mkdir -p $data_path/secret/keys
+    mv $previous_secretkey_path $data_path/secret/keys
+fi
+if [ -f $previous_defaultalias_path ]; then
+    mkdir -p $data_path/secret/keys
+    mv $previous_defaultalias_path $data_path/secret/keys
+fi
+
+# Clean up input dir
+rm -rf ${harbor_prepare_path}/input
 # Create a input dirs
 mkdir -p ${harbor_prepare_path}/input
 input_dir=${harbor_prepare_path}/input
-mkdir -p $input_dir/nginx
-mkdir -p $input_dir/keys
-mkdir -p $input_dir/common
 
-# Copy nginx config file to input dir
+set -e
-cp $ssl_cert_path $input_dir/nginx/server.crt
-cp $ssl_cert_key_path $input_dir/nginx/server.key
-
-# Copy ca bundle to input dir
-if [ -f $registry_custom_ca_bundle ]
-then
-    cp -r $registry_custom_ca_bundle $input_dir/common/custom-ca-bundle.crt
-fi
 
 # Copy harbor.yml to input dir
-cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml
+if [[ ! "$1" =~ ^\-\- ]] && [ -f "$1" ]
+then
+    cp $1 $input_dir/harbor.yml 
+else
+    cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml
+fi
 
 # Create secret dir
 secret_dir=${data_path}/secret
@@ -40,8 +45,8 @@ docker run -it --rm -v $input_dir:/input \
                     -v $harbor_prepare_path:/compose_location \
                     -v $config_dir:/config \
                     -v $secret_dir:/secret \
-                    -v $log_path:/var/log/harbor \
                     goharbor/prepare:dev $@
 
+echo "Clean up the input dir"
 # Clean up input dir
-rm -rf ${harbor_prepare_path}/input
+rm -rf ${harbor_prepare_path}/input

tests/hostcfg.sh

@@ -1,7 +1,9 @@
 #!/bin/bash
 IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'`
-PROTOCOL='https'
 
 #echo $IP
 sudo sed "s/reg.mydomain.com/$IP/" -i make/harbor.yml
-sudo sed "s/^ui_url_protocol: .*/ui_url_protocol: $PROTOCOL/g" -i make/harbor.yml
+
+echo "https:" >> make/harbor.yml
+echo "  certificate: /data/cert/server.crt" >> make/harbor.yml
+echo "  private_key: /data/cert/server.key" >> make/harbor.yml

tests/testprepare.sh

@@ -8,7 +8,7 @@ cp /data/secret/core/private_key.pem /etc/core/
 mkdir src/core/conf
 cp make/common/config/core/app.conf src/core/conf/
 if [ "$(uname)" == "Darwin" ]; then
-	IP=`ifconfig en0 | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}'`
+        IP=`ifconfig en0 | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}'`
 else 
         IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'`
 fi

tests/travis/api_common_install.sh

@@ -10,12 +10,12 @@ sudo sed "s/127.0.0.1/$1/" -i tests/generateCerts.sh
 sudo ./tests/generateCerts.sh
 sudo mkdir -p /etc/docker/certs.d/$1 && sudo cp ./harbor_ca.crt /etc/docker/certs.d/$1/
 
+sudo ./tests/hostcfg.sh
+
 if [ "$2" = 'LDAP' ]; then
-    sudo ./tests/hostcfg.sh LDAP
     cd tests && sudo ./ldapprepare.sh && cd ..
 fi
 
-sudo ./tests/hostcfg.sh
 
 
 # prepare a chart file for API_DB test...