From 93af296eebc086fbe8b6e25bd70adba0a1e30b80 Mon Sep 17 00:00:00 2001 From: Qian Deng Date: Tue, 12 Mar 2019 19:09:01 +0800 Subject: [PATCH] Enhance: refactor the mount dirs and workflow of generate cert mount a temp dir input for all input files and configs generated secrets file stored in data volumns keys dir certs file stored in data volumns nginx dir Signed-off-by: Qian Deng --- Makefile | 2 +- make/harbor.yml | 21 -------- make/photon/prepare/g.py | 19 ++++--- make/photon/prepare/main.py | 25 +++++---- .../prepare/templates/core/config_env.jinja | 1 - .../prepare/templates/core/private_key.pem | 51 ------------------- .../docker_compose/docker-compose.yml.jinja | 23 ++++++--- .../prepare/templates/registry/root.crt | 35 ------------- make/photon/prepare/utils/cert.py | 51 ++++++++++++++----- make/photon/prepare/utils/configs.py | 8 +-- make/photon/prepare/utils/docker_compose.py | 7 +-- make/photon/prepare/utils/notary.py | 8 +-- make/prepare | 51 ++++++++++++++++--- .../cfg/migrator_1_8_0/harbor.yml.tpl | 27 ++-------- tools/migration/migrator.py | 4 +- 15 files changed, 140 insertions(+), 193 deletions(-) delete mode 100644 make/photon/prepare/templates/core/private_key.pem delete mode 100644 make/photon/prepare/templates/registry/root.crt diff --git a/Makefile b/Makefile index ddd8788c0..7c78bcd0b 100644 --- a/Makefile +++ b/Makefile @@ -150,7 +150,7 @@ MIGRATEPATCHBINARYNAME=migrate-patch # configfile CONFIGPATH=$(MAKEPATH) -INSIDE_CONFIGPATH=/harbor_make +INSIDE_CONFIGPATH=/compose_location CONFIGFILE=harbor.yml # prepare parameters diff --git a/make/harbor.yml b/make/harbor.yml index 018390960..351c43cb7 100644 --- a/make/harbor.yml +++ b/make/harbor.yml @@ -48,32 +48,11 @@ log: #only take effect in the first boot, the subsequent changes of these properties #should be performed on web ui -#************************BEGIN INITIAL PROPERTIES************************ - ##The initial password of Harbor admin, only works for the first time when Harbor starts. #It has no effect after the first launch of Harbor. #Change the admin password from UI after launching Harbor. harbor_admin_password: Harbor12345 -##By default the auth mode is db_auth, i.e. the credentials are stored in a local database. -#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server. -auth_mode: db_auth - -#A user's DN who has the permission to search the LDAP/AD server. - -#The base DN from which to look up a user in LDAP/AD -ldap_basedn: ou=people,dc=mydomain,dc=com - -#The attribute used to name a LDAP/AD group, it could be cn, name -ldap_group_gid: cn - -#The flag to control what users have permission to create projects -#The default value "everyone" allows everyone to creates a project. -#Set to "adminonly" so that only admin user can create project. -project_creation_restriction: everyone - -#************************END INITIAL PROPERTIES************************ - ## Harbor DB configuration database: #The address of the Harbor database. Only need to change when using external db. diff --git a/make/photon/prepare/g.py b/make/photon/prepare/g.py index c1f045344..7de8954a7 100644 --- a/make/photon/prepare/g.py +++ b/make/photon/prepare/g.py @@ -1,4 +1,5 @@ import os +from pathlib import Path ## Const DEFAULT_UID = 10000 @@ -7,14 +8,18 @@ DEFAULT_GID = 10000 ## Global variable base_dir = '/harbor_make' templates_dir = "/usr/src/app/templates" -config_dir = os.path.join(base_dir, "common/config") -config_file_path = os.path.join(base_dir, 'harbor.yml') +config_dir = '/config' -private_key_pem_template = os.path.join(templates_dir, "core", "private_key.pem") -root_cert_path_template = os.path.join(templates_dir, "registry", "root.crt") +secret_dir = '/secret' +secret_key_dir='/secret/keys' + +private_key_pem_path = Path('/secret/core/private_key.pem') +root_crt_path = Path('/secret/registry/root.crt') + +config_file_path = '/compose_location/harbor.yml' cert_dir = os.path.join(config_dir, "nginx", "cert") core_cert_dir = os.path.join(config_dir, "core", "certificates") -private_key_pem = os.path.join(config_dir, "core", "private_key.pem") -root_crt = os.path.join(config_dir, "registry", "root.crt") -registry_custom_ca_bundle_config = os.path.join(config_dir, "custom-ca-bundle.crt") \ No newline at end of file + +registry_custom_ca_bundle_storage_path = Path('/secret/common/custom-ca-bundle.crt') +registry_custom_ca_bundle_storage_input_path = Path('/input/common/custom-ca-bundle.crt') \ No newline at end of file diff --git a/make/photon/prepare/main.py b/make/photon/prepare/main.py index 9f67c1659..ac712c860 100644 --- a/make/photon/prepare/main.py +++ b/make/photon/prepare/main.py @@ -2,7 +2,7 @@ import click from utils.misc import delfile from utils.configs import validate, parse_yaml_config -from utils.cert import prepare_ca, SSL_CERT_KEY_PATH, SSL_CERT_PATH, get_secret_key +from utils.cert import prepare_ca, SSL_CERT_KEY_PATH, SSL_CERT_PATH, get_secret_key, copy_ssl_cert, copy_secret_keys from utils.db import prepare_db from utils.jobservice import prepare_job_service from utils.registry import prepare_registry @@ -15,8 +15,8 @@ from utils.clair import prepare_clair from utils.chart import prepare_chartmuseum from utils.docker_compose import prepare_docker_compose from utils.nginx import prepare_nginx, nginx_confd_dir -from g import (config_dir, private_key_pem_template, config_file_path, core_cert_dir, private_key_pem, -root_crt, root_cert_path_template, registry_custom_ca_bundle_config) +from g import (config_dir, config_file_path, core_cert_dir, private_key_pem_path, root_crt_path, +registry_custom_ca_bundle_storage_path, registry_custom_ca_bundle_storage_input_path, secret_key_dir) # Main function @click.command() @@ -38,19 +38,18 @@ def main(conf, with_notary, with_clair, with_chartmuseum): prepare_db(config_dict) prepare_job_service(config_dict) - get_secret_key(config_dict['secretkey_path']) - if config_dict['auth_mode'] == "uaa_auth": - prepare_uaa_cert_file(config_dict['uaa_ca_cert'], core_cert_dir) + copy_secret_keys() + get_secret_key(secret_key_dir) + + if config_dict['protocol'] == 'https': + copy_ssl_cert() # If Customized cert enabled prepare_ca( - customize_crt=config_dict['customize_crt'], - private_key_pem_path=private_key_pem, - private_key_pem_template=private_key_pem_template, - root_crt_path=root_crt, - root_cert_template_path=root_cert_path_template, - registry_custom_ca_bundle_path=config_dict['registry_custom_ca_bundle_path'], - registry_custom_ca_bundle_config=registry_custom_ca_bundle_config) + private_key_pem_path=private_key_pem_path, + root_crt_path=root_crt_path, + registry_custom_ca_bundle_config=registry_custom_ca_bundle_storage_input_path, + registry_custom_ca_bundle_storage_path=registry_custom_ca_bundle_storage_path) if with_notary: prepare_notary(config_dict, nginx_confd_dir, SSL_CERT_PATH, SSL_CERT_KEY_PATH) diff --git a/make/photon/prepare/templates/core/config_env.jinja b/make/photon/prepare/templates/core/config_env.jinja index 7fc84904f..c6485ad36 100644 --- a/make/photon/prepare/templates/core/config_env.jinja +++ b/make/photon/prepare/templates/core/config_env.jinja @@ -1,7 +1,6 @@ PORT=8080 LOG_LEVEL=info EXT_ENDPOINT={{public_url}} -AUTH_MODE={{auth_mode}} SELF_REGISTRATION={{self_registration}} LDAP_URL={{ldap_url}} LDAP_SEARCH_DN={{ldap_searchdn}} diff --git a/make/photon/prepare/templates/core/private_key.pem b/make/photon/prepare/templates/core/private_key.pem deleted file mode 100644 index d2dc85dd1..000000000 --- a/make/photon/prepare/templates/core/private_key.pem +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKAIBAAKCAgEAtpMvyv153iSmwm6TrFpUOzsIGBEDbGtOOEZMEm08D8IC2n1G -d6/XOZ5FxPAD6gIpE0EAcMojY5O0Hl4CDoyV3e/iKcBqFOgYtpogNtan7yT5J8gw -KsPbU/8nBkK75GOq56nfvq4t9GVAclIDtHbuvmlh6O2n+fxtR0M9LbuotbSBdXYU -hzXqiSsMclBvLyIk/z327VP5l0nUNOzPuKIwQjuxYKDkvq1oGy98oVlE6wl0ldh2 -ZYZLGAYbVhqBVUT1Un/PYqi9Nofa2RI5n1WOkUJQp87vb+PUPFhVOdvH/oAzV6/b -9dzyhA5paDM06lj2gsg9hQWxCgbFh1x39c6pSI8hmVe6x2d4tAtSyOm3Qwz+zO2l -bPDvkY8Svh5nxUYObrNreoO8wHr8MC6TGUQLnUt/RfdVKe5fYPFl6VYqJP/L3LDn -Xj771nFq6PKiYbhBwJw3TM49gpKNS/Of70TP2m7nVlyuyMdE5T1j3xyXNkixXqqn -JuSMqX/3Bmm0On9KEbemwn7KRYF/bqc50+RcGUdKNcOkN6vuMVZei4GbxALnVqac -s+/UQAiQP4212UO7iZFwMaCNJ3r/b4GOlyalI1yEA4odoZov7k5zVOzHu8O6QmCj -3R5TVOudpGiUh+lumRRpNqxDgjngLljvaWU6ttyIbjnAwCjnJoppZM2lkRkCAwEA -AQKCAgAvsvCPlf2a3fR7Y6xNISRUfS22K+u7DaXX6fXB8qv4afWY45Xfex89vG35 -78L2Bi55C0h0LztjrpkmPeVHq88TtrJduhl88M5UFpxH93jUb9JwZErBQX4xyb2G -UzUHjEqAT89W3+a9rR5TP74cDd59/MZJtp1mIF7keVqochi3sDsKVxkx4hIuWALe -csk5hTApRyUWCBRzRCSe1yfF0wnMpA/JcP+SGXfTcmqbNNlelo/Q/kaga59+3UmT -C0Wy41s8fIvP+MnGT2QLxkkrqYyfwrWTweqoTtuKEIHjpdnwUcoYJKfQ6jKp8aH0 -STyP5UIyFOKNuFjyh6ZfoPbuT1nGW+YKlUnK4hQ9N/GE0oMoecTaHTbqM+psQvbj -6+CG/1ukA5ZTQyogNyuOApArFBQ+RRmVudPKA3JYygIhwctuB2oItsVEOEZMELCn -g2aVFAVXGfGRDXvpa8oxs3Pc6RJEp/3tON6+w7cMCx0lwN/Jk2Ie6RgTzUycT3k6 -MoTQJRoO6/ZHcx3hTut/CfnrWiltyAUZOsefLuLg+Pwf9GHhOycLRI6gHfgSwdIV -S77UbbELWdscVr1EoPIasUm1uYWBBcFRTturRW+GHJ8TZX+mcWSBcWwBhp15LjEl -tJf+9U6lWMOSB2LvT+vFmR0M9q56fo7UeKFIR7mo7/GpiVu5AQKCAQEA6Qs7G9mw -N/JZOSeQO6xIQakC+sKApPyXO58fa7WQzri+l2UrLNp0DEQfZCujqDgwys6OOzR/ -xg8ZKQWVoad08Ind3ZwoJgnLn6QLENOcE6PpWxA/JjnVGP4JrXCYR98cP0sf9jEI -xkR1qT50GbeqU3RDFliI4kGRvbZ8cekzuWppfQcjstSBPdvuxqAcUVmTnTw83nvD -FmBbhlLiEgI3iKtJ97UB7480ivnWnOuusduk7FO4jF3hkrOa+YRidinTCi8JBo0Y -jx4Ci3Y5x6nvwkXhKzXapd7YmPNisUc5xA7/a+W71cyC0IKUwRc/8pYWLL3R3CpR -YiV8gf6gwzOckQKCAQEAyI9CSNoAQH4zpS8B9PF8zILqEEuun8m1f5JB3hQnfWzm -7uz/zg6I0TkcCE0AJVSKPHQm1V9+TRbF9+DiOWHEYYzPmK8h63SIufaWxZPqai4E -PUj6eQWykBUVJ96n6/AW0JHRZ+WrJ5RXBqCLuY7NP6wDhORrCJjBwaGMohNpbKPS -H3QewsoxCh+CEXKdKyy+/yU/f4E89PlHapkW1/bDJ5u7puSD+KvmiDDIXSBncdOO -uFT8n+XH5IwgjdXFSDim15rQ8jD2l2xLcwKboTpx5GeRl8oB1VGm0fUbBn1dvGPG -4WfHGyrp9VNZtP160WoHr+vRVPqvHNkoeAlCfEwQCQKCAQBN1dtzLN0HgqE8TrOE -ysEDdTCykj4nXNoiJr522hi4gsndhQPLolb6NdKKQW0S5Vmekyi8K4e1nhtYMS5N -5MFRCasZtmtOcR0af87WWucZRDjPmniNCunaxBZ1YFLsRl+H4E6Xir8UgY8O7PYY -FNkFsKIrl3x4nU/RHl8oKKyG9Dyxbq4Er6dPAuMYYiezIAkGjjUCVjHNindnQM2T -GDx2IEe/PSydV6ZD+LguhyU88FCAQmI0N7L8rZJIXmgIcWW0VAterceTHYHaFK2t -u1uB9pcDOKSDnA+Z3kiLT2/CxQOYhQ2clgbnH4YRi/Nm0awsW2X5dATklAKm5GXL -bLSRAoIBAQClaNnPQdTBXBR2IN3pSZ2XAkXPKMwdxvtk+phOc6raHA4eceLL7FrU -y9gd1HvRTfcwws8gXcDKDYU62gNaNhMELWEt2QsNqS/2x7Qzwbms1sTyUpUZaSSL -BohLOKyfv4ThgdIGcXoGi6Z2tcRnRqpq4BCK8uR/05TBgN5+8amaS0ZKYLfaCW4G -nlPk1fVgHWhtAChtnYZLuKg494fKmB7+NMfAbmmVlxjrq+gkPkxyqXvk9Vrg+V8y -VIuozu0Fkouv+GRpyw4ldtCHS1hV0eEK8ow2dwmqCMygDxm58X10mYn2b2PcOTl5 -9sNerUw1GNC8O66K+rGgBk4FKgXmg8kZAoIBABBcuisK250fXAfjAWXGqIMs2+Di -vqAdT041SNZEOJSGNFsLJbhd/3TtCLf29PN/YXtnvBmC37rqryTsqjSbx/YT2Jbr -Bk3jOr9JVbmcoSubXl8d/uzf7IGs91qaCgBwPZHgeH+kK13FCLexz+U9zYMZ78fF -/yO82CpoekT+rcl1jzYn43b6gIklHABQU1uCD6MMyMhJ9Op2WmbDk3X+py359jMc -+Cr2zfzdHAIVff2dOV3OL+ZHEWbwtnn3htKUdOmjoTJrciFx0xNZJS5Q7QYHMONj -yPqbajyhopiN01aBQpCSGF1F1uRpWeIjTrAZPbrwLl9YSYXz0AT05QeFEFk= ------END RSA PRIVATE KEY----- diff --git a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja index 829b24c4b..709cba165 100644 --- a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja +++ b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja @@ -32,7 +32,10 @@ services: volumes: - {{data_volume}}/registry:/storage:z - ./common/config/registry/:/etc/registry/:z - - ./common/config/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z + - {{data_volume}}/secret/registry/root.crt:/etc/registry/root.crt:z +{%if registry_custom_ca_bundle_storage_path %} + - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z +{% endif %} networks: - harbor {% if with_clair %} @@ -121,9 +124,9 @@ services: - SETUID volumes: - ./common/config/core/app.conf:/etc/core/app.conf:z - - ./common/config/core/private_key.pem:/etc/core/private_key.pem:z - ./common/config/core/certificates/:/etc/core/certificates/:z - - {{secretkey_path}}/secretkey:/etc/core/key:z + - {{data_volume}}/secret/core/private_key.pem:/etc/core/private_key.pem:z + - {{data_volume}}/secret/keys/secretkey:/etc/core/key:z - {{data_volume}}/ca_download/:/etc/core/ca/:z - {{data_volume}}/psc/:/etc/core/token/:z - {{data_volume}}/:/data/:z @@ -243,8 +246,10 @@ services: - NET_BIND_SERVICE volumes: - ./common/config/nginx:/etc/nginx:z - - {{cert_key_path}}:/etc/nginx/cert/server.key - - {{cert_path}}:/etc/nginx/cert/server.crt +{% if protocol == 'https' %} + - {{data_volume}}/secret/nginx/server.key:/etc/nginx/cert/server.key + - {{data_volume}}/secret/nginx/server.crt:/etc/nginx/cert/server.crt +{% endif %} networks: - harbor {% if with_notary %} @@ -328,7 +333,9 @@ services: - postgresql volumes: - ./common/config/clair/config.yaml:/etc/clair/config.yaml:z - - ./common/config/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z +{%if registry_custom_ca_bundle_storage_path %} + - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z +{% endif %} logging: driver: "syslog" options: @@ -357,7 +364,9 @@ services: volumes: - {{data_volume}}/chart_storage:/chart_storage:z - ./common/config/chartserver:/etc/chartserver:z - - ./common/config/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z +{%if registry_custom_ca_bundle_storage_path %} + - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z +{% endif %} logging: driver: "syslog" options: diff --git a/make/photon/prepare/templates/registry/root.crt b/make/photon/prepare/templates/registry/root.crt deleted file mode 100644 index c31b27de6..000000000 --- a/make/photon/prepare/templates/registry/root.crt +++ /dev/null @@ -1,35 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIGBzCCA++gAwIBAgIJAKB8CNqCxhr7MA0GCSqGSIb3DQEBCwUAMIGZMQswCQYD -VQQGEwJDTjEOMAwGA1UECAwFU3RhdGUxCzAJBgNVBAcMAkNOMRUwEwYDVQQKDAxv -cmdhbml6YXRpb24xHDAaBgNVBAsME29yZ2FuaXphdGlvbmFsIHVuaXQxFDASBgNV -BAMMC2V4YW1wbGUuY29tMSIwIAYJKoZIhvcNAQkBFhNleGFtcGxlQGV4YW1wbGUu -Y29tMB4XDTE2MDUxNjAyNDY1NVoXDTI2MDUxNDAyNDY1NVowgZkxCzAJBgNVBAYT -AkNOMQ4wDAYDVQQIDAVTdGF0ZTELMAkGA1UEBwwCQ04xFTATBgNVBAoMDG9yZ2Fu -aXphdGlvbjEcMBoGA1UECwwTb3JnYW5pemF0aW9uYWwgdW5pdDEUMBIGA1UEAwwL -ZXhhbXBsZS5jb20xIjAgBgkqhkiG9w0BCQEWE2V4YW1wbGVAZXhhbXBsZS5jb20w -ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC2ky/K/XneJKbCbpOsWlQ7 -OwgYEQNsa044RkwSbTwPwgLafUZ3r9c5nkXE8APqAikTQQBwyiNjk7QeXgIOjJXd -7+IpwGoU6Bi2miA21qfvJPknyDAqw9tT/ycGQrvkY6rnqd++ri30ZUByUgO0du6+ -aWHo7af5/G1HQz0tu6i1tIF1dhSHNeqJKwxyUG8vIiT/PfbtU/mXSdQ07M+4ojBC -O7FgoOS+rWgbL3yhWUTrCXSV2HZlhksYBhtWGoFVRPVSf89iqL02h9rZEjmfVY6R -QlCnzu9v49Q8WFU528f+gDNXr9v13PKEDmloMzTqWPaCyD2FBbEKBsWHXHf1zqlI -jyGZV7rHZ3i0C1LI6bdDDP7M7aVs8O+RjxK+HmfFRg5us2t6g7zAevwwLpMZRAud -S39F91Up7l9g8WXpViok/8vcsOdePvvWcWro8qJhuEHAnDdMzj2Cko1L85/vRM/a -budWXK7Ix0TlPWPfHJc2SLFeqqcm5Iypf/cGabQ6f0oRt6bCfspFgX9upznT5FwZ -R0o1w6Q3q+4xVl6LgZvEAudWppyz79RACJA/jbXZQ7uJkXAxoI0nev9vgY6XJqUj -XIQDih2hmi/uTnNU7Me7w7pCYKPdHlNU652kaJSH6W6ZFGk2rEOCOeAuWO9pZTq2 -3IhuOcDAKOcmimlkzaWRGQIDAQABo1AwTjAdBgNVHQ4EFgQUPJF++WMsv1OJvf7F -oCew37JTnfQwHwYDVR0jBBgwFoAUPJF++WMsv1OJvf7FoCew37JTnfQwDAYDVR0T -BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAb5LvqukMxWd5Zajbh3orfYsXmhWn -UWiwG176+bd3b5xMlG9iLd4vQ11lTZoIhFOfprRQzbizQ8BzR2JBQckpLcy+5hyA -D3M9vLL37OwA0wT6kxFnd6LtlFaH5gG++huw2ts2PDXFz0jqw+0YE/R8ov2+YdaZ -aPSEMunmAuEY1TbYWzz4u6PxycxhQzDQ34ZmJZ34Elvw1NYMfPMGTKp34PsxIcgT -ao5jqb9RMU6JAumfXrOvXRjjl573vX2hgMZzEU6OF2/+uyg95chn6nO1GUQrT2+F -/1xIqfHfFCm8+jujSDgqfBtGI+2C7No+Dq8LEyEINZe6wSQ81+ryt5jy5SZmAsnj -V4OsSIwlpR5fLUwrFStVoUWHEKl1DflkYki/cAC1TL0Om+ldJ219kcOnaXDNaq66 -3I75BvRY7/88MYLl4Fgt7sn05Mn3uNPrCrci8d0R1tlXIcwMdCowIHeZdWHX43f7 -NsVk/7VSOxJ343csgaQc+3WxEFK0tBxGO6GP+Xj0XmdVGLhalVBsEhPjnmx+Yyrn -oMsTA1Yrs88C8ItQn7zuO/30eKNGTnby0gptHiS6sa/c3O083Mpi8y33GPVZDvBl -l9PfSZT8LG7SvpjsdgdNZlyFvTY4vsB+Vd5Howh7gXYPVXdCs4k7HMyo7zvzliZS -ekCw9NGLoNqQqnA= ------END CERTIFICATE----- diff --git a/make/photon/prepare/utils/cert.py b/make/photon/prepare/utils/cert.py index 921c421fb..574f751fc 100644 --- a/make/photon/prepare/utils/cert.py +++ b/make/photon/prepare/utils/cert.py @@ -1,5 +1,6 @@ # Get or generate private key import os, sys, subprocess, shutil +from pathlib import Path from subprocess import DEVNULL from functools import wraps @@ -9,6 +10,17 @@ from .misc import generate_random_string SSL_CERT_PATH = os.path.join("/etc/nginx/cert", "server.crt") SSL_CERT_KEY_PATH = os.path.join("/etc/nginx/cert", "server.key") +input_cert = '/input/nginx/server.crt' +input_cert_key = '/input/nginx/server.key' + +secret_cert_dir = '/secret/nginx' +secret_cert = '/secret/nginx/server.crt' +secret_cert_key = '/secret/nginx/server.key' + +input_secret_keys_dir = '/input/keys' +secret_keys_dir = '/secret/keys' +allowed_secret_key_names = ['defaultalias', 'secretkey'] + def _get_secret(folder, filename, length=16): key_file = os.path.join(folder, filename) if os.path.isfile(key_file): @@ -38,6 +50,20 @@ def get_alias(path): alias = _get_secret(path, "defaultalias", length=8) return alias +def copy_secret_keys(): + if os.path.isdir(secret_cert) and os.path.isdir(input_secret_keys_dir): + input_files = os.listdir(input_secret_keys_dir) + secret_files = os.listdir(secret_keys_dir) + files_need_copy = [x for x in input_files if (x in allowed_secret_key_names) and (x not in secret_files) ] + for f in files_need_copy: + shutil.copy(f, secret_keys_dir) + +def copy_ssl_cert(): + if os.path.isfile(input_cert_key) and os.path.isfile(input_cert): + os.makedirs(secret_cert_dir, exist_ok=True) + shutil.copy(input_cert, secret_cert) + shutil.copy(input_cert_key, secret_cert_key) + ## decorator actions def stat_decorator(func): @wraps(func) @@ -80,22 +106,23 @@ def openssl_installed(): def prepare_ca( - customize_crt, - private_key_pem_path, private_key_pem_template, - root_crt_path, root_cert_template_path, - registry_custom_ca_bundle_path, registry_custom_ca_bundle_config): + private_key_pem_path: Path, + root_crt_path: Path, + registry_custom_ca_bundle_config: Path, + registry_custom_ca_bundle_storage_path: Path): + + if not (private_key_pem_path.exists() and root_crt_path.exists()): + + private_key_pem_path.parent.mkdir(parents=True, exist_ok=True) + root_crt_path.parent.mkdir(parents=True, exist_ok=True) - if (customize_crt == 'on' or customize_crt == True) and openssl_installed(): empty_subj = "/" create_root_cert(empty_subj, key_path=private_key_pem_path, cert_path=root_crt_path) mark_file(private_key_pem_path) mark_file(root_crt_path) - else: - print("Copied configuration file: %s" % private_key_pem_path) - shutil.copyfile(private_key_pem_template, private_key_pem_path) - print("Copied configuration file: %s" % root_crt_path) - shutil.copyfile(root_cert_template_path, root_crt_path) - if len(registry_custom_ca_bundle_path) > 0 and os.path.isfile(registry_custom_ca_bundle_path): - shutil.copyfile(registry_custom_ca_bundle_path, registry_custom_ca_bundle_config) + if not registry_custom_ca_bundle_storage_path.exists() and registry_custom_ca_bundle_config.exists(): + registry_custom_ca_bundle_storage_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(registry_custom_ca_bundle_config, registry_custom_ca_bundle_storage_path) + mark_file(registry_custom_ca_bundle_storage_path) print("Copied custom ca bundle: %s" % registry_custom_ca_bundle_config) \ No newline at end of file diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index 598d7a91d..fae2abbdd 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -7,17 +7,11 @@ def validate(conf, **kwargs): raise Exception( "Error: the protocol must be https when Harbor is deployed with Notary") if protocol == "https": - if not conf.get("cert_path"): ## ssl_path in config + if not conf.get("cert_path"): raise Exception("Error: The protocol is https but attribute ssl_cert is not set") if not conf.get("cert_key_path"): raise Exception("Error: The protocol is https but attribute ssl_cert_key is not set") - # Project validate - project_creation = conf.get("project_creation_restriction") - if project_creation != "everyone" and project_creation != "adminonly": - raise Exception( - "Error invalid value for project_creation_restriction: %s" % project_creation) - # Storage validate valid_storage_drivers = ["filesystem", "azure", "gcs", "s3", "swift", "oss"] storage_provider_name = conf.get("storage_provider_name") diff --git a/make/photon/prepare/utils/docker_compose.py b/make/photon/prepare/utils/docker_compose.py index be86575d2..84e972900 100644 --- a/make/photon/prepare/utils/docker_compose.py +++ b/make/photon/prepare/utils/docker_compose.py @@ -1,6 +1,6 @@ import os -from g import base_dir, templates_dir +from g import templates_dir from .jinja import render_jinja @@ -17,7 +17,7 @@ NGINX_VERSION = VERSION_TAG # version of chartmuseum docker_compose_template_path = os.path.join(templates_dir, 'docker_compose', 'docker-compose.yml.jinja') -docker_compose_yml_path = os.path.join(base_dir, 'docker-compose.yml') +docker_compose_yml_path = '/compose_location/docker-compose.yml' def check_configs(configs): pass @@ -36,10 +36,11 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum): 'log_location': configs['log_location'], 'cert_key_path': configs['cert_key_path'], 'cert_path': configs['cert_path'], + 'protocol': configs['protocol'], + 'registry_custom_ca_bundle_storage_path': configs['registry_custom_ca_bundle_path'], 'with_notary': with_notary, 'with_clair': with_clair, 'with_chartmuseum': with_chartmuseum } - rendering_variables['secretkey_path'] = configs['secretkey_path'] render_jinja(docker_compose_template_path, docker_compose_yml_path, **rendering_variables) \ No newline at end of file diff --git a/make/photon/prepare/utils/notary.py b/make/photon/prepare/utils/notary.py index 934deb832..6944a90c4 100644 --- a/make/photon/prepare/utils/notary.py +++ b/make/photon/prepare/utils/notary.py @@ -1,5 +1,5 @@ import os, shutil -from g import base_dir, templates_dir, config_dir, root_crt, DEFAULT_UID, DEFAULT_GID +from g import base_dir, templates_dir, config_dir, root_crt_path, secret_key_dir,DEFAULT_UID, DEFAULT_GID from .cert import openssl_installed, create_cert, create_root_cert, get_alias from .jinja import render_jinja from .misc import mark_file, prepare_config_dir @@ -23,7 +23,7 @@ def prepare_env_notary(customize_crt, nginx_config_dir): notary_config_dir = prepare_config_dir(config_dir, "notary") if (customize_crt == 'on' or customize_crt == True) and openssl_installed(): try: - temp_cert_dir = os.path.join(base_dir, "cert_tmp") + temp_cert_dir = os.path.join('/tmp', "cert_tmp") if not os.path.exists(temp_cert_dir): os.makedirs(temp_cert_dir) ca_subj = "/C=US/ST=California/L=Palo Alto/O=GoHarbor/OU=Harbor/CN=Self-signed by GoHarbor" @@ -50,7 +50,7 @@ def prepare_env_notary(customize_crt, nginx_config_dir): shutil.copy2(os.path.join(notary_template_dir, "notary-signer.key"), notary_config_dir) shutil.copy2(os.path.join(notary_template_dir, "notary-signer-ca.crt"), notary_config_dir) - shutil.copy2(root_crt, notary_config_dir) + shutil.copy2(root_crt_path, notary_config_dir) shutil.copy2( os.path.join(notary_template_dir, "server_env.jinja"), os.path.join(notary_config_dir, "server_env")) @@ -95,7 +95,7 @@ def prepare_notary(config_dict, nginx_config_dir, ssl_cert_path, ssl_cert_key_pa ssl_cert=ssl_cert_path, ssl_cert_key=ssl_cert_key_path) - default_alias = get_alias(config_dict['secretkey_path']) + default_alias = get_alias(secret_key_dir) render_jinja( notary_signer_env_template, notary_signer_env_path, diff --git a/make/prepare b/make/prepare index 4dcec6992..e6e94eb09 100755 --- a/make/prepare +++ b/make/prepare @@ -1,10 +1,49 @@ #!/bin/bash -host_make_path="$( cd "$(dirname "$0")" ; pwd -P )" +# If compling source code this dir is harbor's make dir +# If install harbor via pacakge, this dir is harbor's root dir +harbor_prepare_path="$( cd "$(dirname "$0")" ; pwd -P )" -echo host make path is set to ${host_make_path} -data_path=$(grep '^[^#]*data_volume' ${host_make_path}/harbor.yml | awk '{print $NF}') -log_path=$(grep '^[^#]*location' ${host_make_path}/harbor.yml | awk '{print $NF}') -secretkey_path=$(grep '^[^#]*secretkey_path' ${host_make_path}/harbor.yml | awk '{print $NF}') +echo host make path is set to ${harbor_prepare_path} +data_path=$(grep '^[^#]*data_volume:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') +log_path=$(grep '^[^#]*location:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') +secretkey_path=$(grep '^[^#]*secretkey_path:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') +ssl_cert_path=$(grep '^[^#]*ssl_cert:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') +ssl_cert_key_path=$(grep '^[^#]*ssl_cert_key:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') +registry_custom_ca_bundle=$(grep '^[^#]*registry_custom_ca_bundle:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') +# Create a input dirs +mkdir -p ${harbor_prepare_path}/input +input_dir=${harbor_prepare_path}/input +mkdir -p $input_dir/nginx +mkdir -p $input_dir/keys +mkdir -p $input_dir/common -docker run -it --rm -v ${host_make_path}:/harbor_make -v $data_path:/data -v $log_path:/var/log/harbor -v $secretkey_path:$secretkey_path goharbor/prepare:1.7.1 $@ +# Create secret dir +secret_dir=${data_path}/secret +config_dir=$harbor_prepare_path/common/config + +# Copy nginx config file to input dir +cp $ssl_cert_path $input_dir/nginx/server.crt +cp $ssl_cert_key_path $input_dir/nginx/server.key + +# Copy secretkey to input dir +cp -r $secretkey_path $input_dir/keys + +# Copy ca bundle to input dir +if [ -f $registry_custom_ca_bundle ] +then + cp -r $registry_custom_ca_bundle $input_dir/common/custom-ca-bundle.crt +fi + +# Copy harbor.yml to input dir +cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml + +docker run -it --rm -v $input_dir:/input \ + -v $harbor_prepare_path:/compose_location \ + -v $config_dir:/config \ + -v $secret_dir:/secret \ + -v $log_path:/var/log/harbor \ + goharbor/prepare:1.7.1 $@ + +# Clean up input dir +rm -rf ${harbor_prepare_path}/input \ No newline at end of file diff --git a/tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl b/tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl index 6ec529dba..7716c5fe1 100644 --- a/tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl +++ b/tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl @@ -48,29 +48,10 @@ log: #only take effect in the first boot, the subsequent changes of these properties #should be performed on web ui -#************************BEGIN INITIAL PROPERTIES************************ - -##By default the auth mode is db_auth, i.e. the credentials are stored in a local database. -#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server. -auth_mode: db_auth - -#The base DN from which to look up a user in LDAP/AD -ldap_basedn: ou=people,dc=mydomain,dc=com - -#The attribute used to name a LDAP/AD group, it could be cn, name -ldap_group_gid: cn - -#The following attributes only need to be set when auth mode is uaa_auth -uaa_ca_cert: $uaa_ca_cert - -#The flag to control what users have permission to create projects -#The default value "everyone" allows everyone to creates a project. -#Set to "adminonly" so that only admin user can create project. -project_creation_restriction: everyone - -#************************END INITIAL PROPERTIES************************ - - +##The initial password of Harbor admin, only works for the first time when Harbor starts. +#It has no effect after the first launch of Harbor. +#Change the admin password from UI after launching Harbor. +harbor_admin_password: Harbor12345 database: #The address of the Harbor database. Only need to change when using external db. diff --git a/tools/migration/migrator.py b/tools/migration/migrator.py index f58bef0af..595fa15b2 100644 --- a/tools/migration/migrator.py +++ b/tools/migration/migrator.py @@ -71,7 +71,7 @@ class CfgMigrator(): copyfile(self.cfg_path, self.backup_path+"/harbor.cfg") print ("Success to backup harbor.cfg.") return True - except Exception, e: + except Exception as e: print ("Back up error: %s" % str(e)) return False @@ -83,7 +83,7 @@ class CfgMigrator(): copyfile(self.backup_path+"/harbor.cfg", self.cfg_path) print ("Success to restore harbor.cfg.") return True - except Exception, e: + except Exception as e: print ("Restore error: %s" % str(e)) return False