diff --git a/make/harbor.yml b/make/harbor.yml index d38e973b0..dd7dd65ab 100644 --- a/make/harbor.yml +++ b/make/harbor.yml @@ -127,11 +127,8 @@ _version: 1.7.0 # Umcomments if using external Redis server # external_redis: -# # Redis connection address # host: redis -# # Redis connection port # port: 6379 -# # Redis connection password # password: # # db_index 0 is for core, it's unchangeable # registry_db_index: 1 diff --git a/make/photon/prepare/templates/core/config_env.jinja b/make/photon/prepare/templates/core/config_env.jinja index a2f882bbb..e9801e539 100644 --- a/make/photon/prepare/templates/core/config_env.jinja +++ b/make/photon/prepare/templates/core/config_env.jinja @@ -1,16 +1,6 @@ PORT=8080 LOG_LEVEL=info EXT_ENDPOINT={{public_url}} -SELF_REGISTRATION={{self_registration}} -LDAP_URL={{ldap_url}} -LDAP_SEARCH_DN={{ldap_searchdn}} -LDAP_SEARCH_PWD={{ldap_search_pwd}} -LDAP_BASE_DN={{ldap_basedn}} -LDAP_FILTER={{ldap_filter}} -LDAP_UID={{ldap_uid}} -LDAP_SCOPE={{ldap_scope}} -LDAP_TIMEOUT={{ldap_timeout}} -LDAP_VERIFY_CERT={{ldap_verify_cert}} DATABASE_TYPE=postgresql POSTGRESQL_HOST={{db_host}} POSTGRESQL_PORT={{db_port}} @@ -18,49 +8,29 @@ POSTGRESQL_USERNAME={{db_user}} POSTGRESQL_PASSWORD={{db_password}} POSTGRESQL_DATABASE=registry POSTGRESQL_SSLMODE=disable -LDAP_GROUP_BASEDN={{ldap_group_basedn}} -LDAP_GROUP_FILTER={{ldap_group_filter}} -LDAP_GROUP_GID={{ldap_group_gid}} -LDAP_GROUP_SCOPE={{ldap_group_scope}} REGISTRY_URL={{registry_url}} TOKEN_SERVICE_URL={{token_service_url}} -EMAIL_HOST={{email_host}} -EMAIL_PORT={{email_port}} -EMAIL_USR={{email_usr}} -EMAIL_PWD={{email_pwd}} -EMAIL_SSL={{email_ssl}} -EMAIL_FROM={{email_from}} -EMAIL_IDENTITY={{email_identity}} -EMAIL_INSECURE={{email_insecure}} HARBOR_ADMIN_PASSWORD={{harbor_admin_password}} -PROJECT_CREATION_RESTRICTION={{project_creation_restriction}} MAX_JOB_WORKERS={{max_job_workers}} CORE_SECRET={{core_secret}} JOBSERVICE_SECRET={{jobservice_secret}} -TOKEN_EXPIRATION={{token_expiration}} CFG_EXPIRATION=5 ADMIRAL_URL={{admiral_url}} WITH_NOTARY={{with_notary}} WITH_CLAIR={{with_clair}} -CLAIR_DB_PASSWORD={{clair_db_password}} -CLAIR_DB_HOST={{clair_db_host}} -CLAIR_DB_PORT={{clair_db_port}} -CLAIR_DB_USERNAME={{clair_db_username}} +CLAIR_DB_PASSWORD={{db_password}} +CLAIR_DB_HOST={{db_host}} +CLAIR_DB_PORT={{db_port}} +CLAIR_DB_USERNAME={{db_user}} CLAIR_DB={{clair_db}} CLAIR_DB_SSLMODE=disable -UAA_ENDPOINT={{uaa_endpoint}} -UAA_CLIENTID={{uaa_clientid}} -UAA_CLIENTSECRET={{uaa_clientsecret}} -UAA_VERIFY_CERT={{uaa_verify_cert}} CORE_URL={{core_url}} JOBSERVICE_URL={{jobservice_url}} CLAIR_URL={{clair_url}} NOTARY_URL={{notary_url}} REGISTRY_STORAGE_PROVIDER_NAME={{storage_provider_name}} READ_ONLY=false -SKIP_RELOAD_ENV_PATTERN={{skip_reload_env_pattern}} RELOAD_KEY={{reload_key}} CHART_REPOSITORY_URL={{chart_repository_url}} -LDAP_GROUP_ADMIN_DN={{ldap_group_admin_dn}} REGISTRY_CONTROLLER_URL={{registry_controller_url}} WITH_CHARTMUSEUM={{with_chartmuseum}} diff --git a/make/photon/prepare/utils/clair.py b/make/photon/prepare/utils/clair.py index 3a3197aee..86f49abd3 100644 --- a/make/photon/prepare/utils/clair.py +++ b/make/photon/prepare/utils/clair.py @@ -27,17 +27,17 @@ def prepare_clair(config_dict): render_jinja( postgres_env_template, postgres_env_path, - password=config_dict['clair_db_password']) + password=config_dict['db_password']) render_jinja( clair_config_template, clair_config_path, uid=DEFAULT_UID, gid=DEFAULT_GID, - password= config_dict['clair_db_password'], - username= config_dict['clair_db_username'], - host= config_dict['clair_db_host'], - port= config_dict['clair_db_port'], + password= config_dict['db_password'], + username= config_dict['db_user'], + host= config_dict['db_host'], + port= config_dict['db_port'], dbname= config_dict['clair_db'], interval= config_dict['clair_updaters_interval']) diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index 3df0ed768..cadd79030 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -37,10 +37,6 @@ def validate(conf, **kwargs): raise Exception( "Error: redis_port in harbor.cfg needs to point to the port of Redis server or cluster.") - redis_db_index = conf.get("redis_db_index") - if len(redis_db_index.split(",")) != 3: - raise Exception( - "Error invalid value for redis_db_index: %s. please set it as 1,2,3" % redis_db_index) def parse_versions(): if not versions_file_path.is_file(): @@ -74,116 +70,82 @@ def parse_yaml_config(config_file_path): config_dict['protocol'] = configs.get("ui_url_protocol") config_dict['public_url'] = config_dict['protocol'] + "://" + config_dict['hostname'] - # Data path volume - config_dict['data_volume'] = configs.get("data_volume") - - # Email related configs - config_dict['email_identity'] = configs.get("email_identity") - config_dict['email_host'] = configs.get("email_server") - config_dict['email_port'] = configs.get("email_server_port") - config_dict['email_usr'] = configs.get("email_username") - config_dict['email_pwd'] = configs.get("email_password") - config_dict['email_from'] = configs.get("email_from") - config_dict['email_ssl'] = configs.get("email_ssl") - config_dict['email_insecure'] = configs.get("email_insecure") - config_dict['harbor_admin_password'] = configs.get("harbor_admin_password") - config_dict['auth_mode'] = configs.get("auth_mode") - config_dict['ldap_url'] = configs.get("ldap_url") - - # LDAP related configs - # this two options are either both set or unset - if configs.get("ldap_searchdn"): - config_dict['ldap_searchdn'] = configs["ldap_searchdn"] - config_dict['ldap_search_pwd'] = configs["ldap_search_pwd"] - else: - config_dict['ldap_searchdn'] = "" - config_dict['ldap_search_pwd'] = "" - config_dict['ldap_basedn'] = configs.get("ldap_basedn") - # ldap_filter is null by default - if configs.get("ldap_filter"): - config_dict['ldap_filter'] = configs["ldap_filter"] - else: - config_dict['ldap_filter'] = "" - config_dict['ldap_uid'] = configs.get("ldap_uid") - config_dict['ldap_scope'] = configs.get("ldap_scope") - config_dict['ldap_timeout'] = configs.get("ldap_timeout") - config_dict['ldap_verify_cert'] = configs.get("ldap_verify_cert") - config_dict['ldap_group_basedn'] = configs.get("ldap_group_basedn") - config_dict['ldap_group_filter'] = configs.get("ldap_group_filter") - config_dict['ldap_group_gid'] = configs.get("ldap_group_gid") - config_dict['ldap_group_scope'] = configs.get("ldap_group_scope") - # Admin dn - config_dict['ldap_group_admin_dn'] = configs.get("ldap_group_admin_dn") or '' - - # DB configs - db_configs = configs.get('database') - config_dict['db_host'] = db_configs.get("host") - config_dict['db_port'] = db_configs.get("port") - config_dict['db_user'] = db_configs.get("username") - config_dict['db_password'] = db_configs.get("password") - - config_dict['self_registration'] = configs.get("self_registration") - config_dict['project_creation_restriction'] = configs.get("project_creation_restriction") # secure configs if config_dict['protocol'] == "https": config_dict['cert_path'] = configs.get("ssl_cert") config_dict['cert_key_path'] = configs.get("ssl_cert_key") - config_dict['customize_crt'] = configs.get("customize_crt") - config_dict['max_job_workers'] = configs.get("max_job_workers") - config_dict['token_expiration'] = configs.get("token_expiration") - config_dict['secretkey_path'] = configs["secretkey_path"] - # Admiral configs - if configs.get("admiral_url"): - config_dict['admiral_url'] = configs["admiral_url"] - else: - config_dict['admiral_url'] = "" + + # DB configs + db_configs = configs.get('database') + if db_configs: + config_dict['db_host'] = 'postgresql' + config_dict['db_port'] = 5432 + config_dict['db_user'] = 'postgres' + config_dict['db_password'] = db_configs.get("password") or 'root123' + config_dict['ssl_mode'] = 'disable' + + + # Data path volume + config_dict['data_volume'] = configs.get('data_volume') + + # Initial Admin Password + config_dict['harbor_admin_password'] = configs.get("harbor_admin_password") + + # Registry storage configs + storage_config = configs.get('storage') or {} + config_dict['storage_provider_name'] = storage_config.get("registry_storage_provider_name") or '' + config_dict['storage_provider_config'] = storage_config.get("registry_storage_provider_config") or '' + # yaml requires 1 or more spaces between the key and value + config_dict['storage_provider_config'] = config_dict['storage_provider_config'].replace(":", ": ", 1) + config_dict['registry_custom_ca_bundle_path'] = storage_config.get("registry_custom_ca_bundle") or '' + # Clair configs clair_configs = configs.get("clair") or {} - config_dict['clair_db_password'] = clair_configs.get("db_password") or '' - config_dict['clair_db_host'] = clair_configs.get("db_host") or '' - config_dict['clair_db_port'] = clair_configs.get("db_port") or '' - config_dict['clair_db_username'] = clair_configs.get("db_username") or '' - config_dict['clair_db'] = clair_configs.get("db") or '' - config_dict['clair_updaters_interval'] = clair_configs.get("updaters_interval") or '' + config_dict['clair_db'] = 'postgres' + config_dict['clair_updaters_interval'] = clair_configs.get("updaters_interval") or 12 config_dict['clair_http_proxy'] = clair_configs.get('http_proxy') or '' config_dict['clair_https_proxy'] = clair_configs.get('https_proxy') or '' config_dict['clair_no_proxy'] = clair_configs.get('no_proxy') or '' - # UAA configs - config_dict['uaa_endpoint'] = configs.get("uaa_endpoint") - config_dict['uaa_clientid'] = configs.get("uaa_clientid") - config_dict['uaa_clientsecret'] = configs.get("uaa_clientsecret") - config_dict['uaa_verify_cert'] = configs.get("uaa_verify_cert") - config_dict['uaa_ca_cert'] = configs.get("uaa_ca_cert") + + # jobservice config + config_dict['max_job_workers'] = configs.get("max_job_workers") + config_dict['jobservice_secret'] = generate_random_string(16) + # Log configs log_configs = configs.get('log') or {} config_dict['log_location'] = log_configs.get("location") config_dict['log_rotate_count'] = log_configs.get("rotate_count") config_dict['log_rotate_size'] = log_configs.get("rotate_size") + config_dict['log_level'] = log_configs.get('level') - # Redis configs - redis_configs = configs.get("redis") - if redis_configs: - config_dict['redis_host'] = redis_configs.get("host") or '' - config_dict['redis_port'] = redis_configs.get("port") or '' - config_dict['redis_password'] = redis_configs.get("password") or '' - config_dict['redis_db_index'] = redis_configs.get("db_index") or '' - db_indexs = config_dict['redis_db_index'].split(',') - config_dict['redis_db_index_reg'] = db_indexs[0] - config_dict['redis_db_index_js'] = db_indexs[1] - config_dict['redis_db_index_chart'] = db_indexs[2] - else: - config_dict['redis_host'] = '' - config_dict['redis_port'] = '' - config_dict['redis_password'] = '' - config_dict['redis_db_index'] = '' - config_dict['redis_db_index_reg'] = '' - config_dict['redis_db_index_js'] = '' - config_dict['redis_db_index_chart'] = '' + + # external DB, if external_db enabled, it will cover the database config + external_db_configs = configs.get('external_database') + if external_db_configs: + config_dict['db_password'] = external_db_configs.get('password') or 'root123' + if external_db_configs.get('host'): + config_dict['db_host'] = external_db_configs['host'] + if external_db_configs.get('port'): + config_dict['db_port'] = external_db_configs['port'] + if external_db_configs.get('username'): + config_dict['db_user'] = db_configs['username'] + if external_db_configs.get('ssl_mode'): + config_dict['db_ssl_mode'] = external_db_configs['ssl_mode'] + + + # external_redis configs + redis_configs = configs.get("external_redis") or {} + config_dict['redis_host'] = redis_configs.get("host") or 'redis' + config_dict['redis_port'] = redis_configs.get("port") or 6379 + config_dict['redis_password'] = redis_configs.get("password") or '' + config_dict['redis_db_index_reg'] = redis_configs.get('registry_db_index') or 1 + config_dict['redis_db_index_js'] = redis_configs.get('jobservice_db_index') or 2 + config_dict['redis_db_index_chart'] = redis_configs.get('chartmuseum_db_index') or 3 # redis://[arbitrary_username:password@]ipaddress:port/database_index if config_dict.get('redis_password'): @@ -193,26 +155,14 @@ def parse_yaml_config(config_file_path): config_dict['redis_url_js'] = "redis://%s:%s/%s" % (config_dict['redis_host'], config_dict['redis_port'], config_dict['redis_db_index_js']) config_dict['redis_url_reg'] = "redis://%s:%s/%s" % (config_dict['redis_host'], config_dict['redis_port'], config_dict['redis_db_index_reg']) - if configs.get("skip_reload_env_pattern"): - config_dict['skip_reload_env_pattern'] = configs["skip_reload_env_pattern"] - else: - config_dict['skip_reload_env_pattern'] = "$^" - # Registry storage configs - storage_config = configs.get('storage') - if storage_config: - config_dict['storage_provider_name'] = storage_config.get("registry_storage_provider_name") or '' - config_dict['storage_provider_config'] = storage_config.get("registry_storage_provider_config") or '' - # yaml requires 1 or more spaces between the key and value - config_dict['storage_provider_config'] = config_dict['storage_provider_config'].replace(":", ": ", 1) - config_dict['registry_custom_ca_bundle_path'] = storage_config.get("registry_custom_ca_bundle") or '' - else: - config_dict['storage_provider_name'] = '' - config_dict['storage_provider_config'] = '' - config_dict['registry_custom_ca_bundle_path'] = '' - - # auto generate secret string + # auto generated secret string for core config_dict['core_secret'] = generate_random_string(16) - config_dict['jobservice_secret'] = generate_random_string(16) + + # Admiral configs + if configs.get("admiral_url"): + config_dict['admiral_url'] = configs["admiral_url"] + else: + config_dict['admiral_url'] = "" return config_dict \ No newline at end of file diff --git a/make/photon/prepare/utils/misc.py b/make/photon/prepare/utils/misc.py index 1e1f05db1..fe6bcc7f8 100644 --- a/make/photon/prepare/utils/misc.py +++ b/make/photon/prepare/utils/misc.py @@ -36,13 +36,6 @@ def validate(conf, **kwargs): raise Exception( "Error: The path for certificate key: %s is invalid" % cert_key_path) - # Project validate - project_creation = conf.get( - "configuration", "project_creation_restriction") - if project_creation != "everyone" and project_creation != "adminonly": - raise Exception( - "Error invalid value for project_creation_restriction: %s" % project_creation) - # Storage validate valid_storage_drivers = ["filesystem", "azure", "gcs", "s3", "swift", "oss"] diff --git a/make/photon/prepare/utils/notary.py b/make/photon/prepare/utils/notary.py index 041d46be1..db791ea64 100644 --- a/make/photon/prepare/utils/notary.py +++ b/make/photon/prepare/utils/notary.py @@ -19,7 +19,7 @@ notary_signer_env_path = os.path.join(notary_config_dir, "signer_env") notary_server_env_path = os.path.join(notary_config_dir, "server_env") -def prepare_env_notary(customize_crt, nginx_config_dir): +def prepare_env_notary(nginx_config_dir): notary_config_dir = prepare_config_dir(config_dir, "notary") old_signer_cert_secret_path = pathlib.Path(os.path.join(config_dir, 'notary-signer.crt')) old_signer_key_secret_path = pathlib.Path(os.path.join(config_dir, 'notary-signer.key')) @@ -87,7 +87,7 @@ def prepare_env_notary(customize_crt, nginx_config_dir): def prepare_notary(config_dict, nginx_config_dir, ssl_cert_path, ssl_cert_key_path): - prepare_env_notary(config_dict['customize_crt'], nginx_config_dir) + prepare_env_notary(nginx_config_dir) render_jinja( notary_signer_pg_template, diff --git a/make/photon/prepare/utils/uaa.py b/make/photon/prepare/utils/uaa.py deleted file mode 100644 index 151bba54f..000000000 --- a/make/photon/prepare/utils/uaa.py +++ /dev/null @@ -1,11 +0,0 @@ -import os, shutil - -def prepare_uaa_cert_file(uaa_ca_cert, core_cert_dir): - if os.path.isfile(uaa_ca_cert): - if not os.path.isdir(core_cert_dir): - os.makedirs(core_cert_dir) - core_uaa_ca = os.path.join(core_cert_dir, "uaa_ca.pem") - print("Copying UAA CA cert to %s" % core_uaa_ca) - shutil.copyfile(uaa_ca_cert, core_uaa_ca) - else: - print("Can not find UAA CA cert: %s, skip" % uaa_ca_cert) \ No newline at end of file diff --git a/make/prepare b/make/prepare index 5753bb637..8907087d6 100755 --- a/make/prepare +++ b/make/prepare @@ -7,7 +7,6 @@ harbor_prepare_path="$( cd "$(dirname "$0")" ; pwd -P )" echo host make path is set to ${harbor_prepare_path} data_path=$(grep '^[^#]*data_volume:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') log_path=$(grep '^[^#]*location:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') -secretkey_path=$(grep '^[^#]*secretkey_path:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') ssl_cert_path=$(grep '^[^#]*ssl_cert:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') ssl_cert_key_path=$(grep '^[^#]*ssl_cert_key:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') registry_custom_ca_bundle=$(grep '^[^#]*registry_custom_ca_bundle:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') @@ -23,9 +22,6 @@ mkdir -p $input_dir/common cp $ssl_cert_path $input_dir/nginx/server.crt cp $ssl_cert_key_path $input_dir/nginx/server.key -# Copy secretkey to input dir -cp -r $secretkey_path $input_dir/keys - # Copy ca bundle to input dir if [ -f $registry_custom_ca_bundle ] then