diff --git a/CHANGELOG.md b/CHANGELOG.md index fd9844afa..ffc447181 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## v1.7.5 (2019-04-02) +* Bumped up Clair to v2.0.8 +* Fixed issues in supporting windows images. #6992 #6369 +* Removed user-agent check-in notification handler. #5729 +* Fixed the issue global search not working if chartmusuem is not installed #6753 + +## v1.7.4 (2019-03-04) +[Full list of issues fixed in v1.7.4](https://github.com/goharbor/harbor/issues?q=is%3Aissue+is%3Aclosed+label%3Atarget%2F1.7.4) + +## v1.7.1 (2019-01-07) +[Full list of issues fixed in v1.7.1](https://github.com/goharbor/harbor/issues?q=is%3Aissue+is%3Aclosed+label%3Atarget%2F1.7.1) + +## v1.7.0 (2018-12-19) +* Support deploy Harbor with Helm Chart, enables the user to have high availability of Harbor services, refer to the [Installation and Configuration Guide](https://github.com/goharbor/harbor-helm/tree/1.0.0). +* Support on-demand Garbage Collection, enables the admin to configure run docker registry garbage collection manually or automatically with a cron schedule. +* Support Image Retag, enables the user to tag image to different repositories and projects, this is particularly useful in cases when images need to be retagged programmatically in a CI pipeline. +* Support Image Build History, makes it easy to see the contents of a container image, refer to the [User Guide](https://github.com/goharbor/harbor/blob/release-1.7.0/docs/user_guide.md#build-history). +* Support Logger customization, enables the user to customize STDOUT / STDERR / FILE / DB logger of running jobs. +* Improve user experience of Helm Chart Repository: + - Chart searching included in the global search results + - Show chart versions total number in the chart list + - Mark labels to helm charts + - The latest version can be downloaded as default one on the chart list view + - The chart can be deleted by deleting all the versions under it + + ## v1.6.0 (2018-09-11) - Support manages Helm Charts: From version 1.6.0, Harbor is upgraded to be a composite cloud-native registry, which supports both image management and helm charts management. diff --git a/Makefile b/Makefile index 93787f395..d238cf9ab 100644 --- a/Makefile +++ b/Makefile @@ -295,7 +295,7 @@ compile: check_environment versions_prepare compile_core compile_jobservice comp update_prepare_version: @echo "substitude the prepare version tag in prepare file..." - $(SEDCMD) -i -e 's/goharbor\/prepare:.*[[:space:]]\+/goharbor\/prepare:$(VERSIONTAG) /' $(MAKEPATH)/prepare ; + @$(SEDCMD) -i -e 's/goharbor\/prepare:.*[[:space:]]\+/goharbor\/prepare:$(VERSIONTAG) /' $(MAKEPATH)/prepare ; prepare: update_prepare_version @echo "preparing..." @@ -310,7 +310,7 @@ build: install: compile ui_version build prepare start -package_online: prepare +package_online: update_prepare_version @echo "packing online package ..." @cp -r make $(HARBORPKG) @if [ -n "$(REGISTRYSERVER)" ] ; then \ diff --git a/make/harbor.yml b/make/harbor.yml index c4e9db140..7abb44217 100644 --- a/make/harbor.yml +++ b/make/harbor.yml @@ -1,114 +1,93 @@ ## Configuration file of Harbor -#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! -_version: 1.7.0 #The IP address or hostname to access admin UI and registry service. #DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. -#DO NOT comment out this line, modify the value of "hostname" directly, or the installation will fail. hostname: reg.mydomain.com +# core, harbor +http: + port: 80 -#The protocol for accessing the UI and token/notification service, by default it is http. -#It can be set to https if ssl is enabled on nginx. -ui_url_protocol: https +# https: +# port: 443 +# #The path of cert and key files for nginx +# certificate: /your/certificate/path +# private_key: /your/private/key/path -#Maximum number of job workers in job service -max_job_workers: 10 +# Uncomment extearnal_url if you want to enable external proxy +# And when it enabled the hostname will no longger used +# external_url: https://reg.mydomain.com:8433 + +# The initial password of Harbor admin +# It only works in first time to install harbor +# Remember Change the admin password from UI after launching Harbor. +harbor_admin_password: Harbor12345 + +## Harbor DB configuration +database: + #The password for the root user of Harbor DB. Change this before any production use. + password: root123 # The default data volume data_volume: /data -#The path of cert and key files for nginx, they are applied only the protocol is set to https -ssl_cert: /data/cert/server.crt -ssl_cert_key: /data/cert/server.key +# Harbor Storage settings by default is using /data dir on local filesystem +# Uncomment storage_service setting If you want to using external storage +# storage_service: +# # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore +# # of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signed certificate. +# ca_bundle: -#The path of secretkey storage -secretkey_path: /data +# # storage backend, default is filesystem, options include filesystem, azure, gcs, s3, swift and oss +# # for more info about this configuration please refer https://docs.docker.com/registry/configuration/ +# filesystem: +# maxthreads: 100 -#Admiral's url, comment this attribute, or set its value to NA when Harbor is standalone -admiral_url: NA +# Clair configuration +clair: + # The interval of clair updaters, the unit is hour, set to 0 to disable the updaters. + updaters_interval: 12 + + # Config http proxy for Clair, e.g. http://my.proxy.com:3128 + # Clair doesn't need to connect to harbor internal components via http proxy. + http_proxy: + https_proxy: + no_proxy: 127.0.0.1,localhost,core,registry + +jobservice: + # Maximum number of job workers in job service + max_job_workers: 10 # Log configurations log: + # options are debug, info, warn, error + level: info # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. rotate_count: 50 # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G # are all valid. rotate_size: 200M - # The directory that store log files + # The directory on your host that store log location: /var/log/harbor -#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES -#only take effect in the first boot, the subsequent changes of these properties -#should be performed on web ui +#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! +_version: 1.7.0 -##The initial password of Harbor admin, only works for the first time when Harbor starts. -#It has no effect after the first launch of Harbor. -#Change the admin password from UI after launching Harbor. -harbor_admin_password: Harbor12345 +# Uncomment external_database if using external database. And the password will replace the the password setting in database. +# And currently ontly support postgres. +# external_database: +# host: postgresql +# port: 5432 +# username: postgres +# password: root123 +# ssl_mode: disable -## Harbor DB configuration -database: - #The address of the Harbor database. Only need to change when using external db. - host: postgresql - #The port of Harbor database host - port: 5432 - #The user name of Harbor database - username: postgres - #The password for the root user of Harbor DB. Change this before any production use. - password: root123 - - -# Redis server configuration -redis: - # Redis connection address - host: redis - # Redis connection port - port: 6379 - # Redis connection password - password: - # Redis connection db index - # db_index 1,2,3 is for registry, jobservice and chartmuseum. - # db_index 0 is for UI, it's unchangeable - db_index: 1,2,3 - - -# Clair DB configuration -clair: - # Clair DB host address. Only change it when using an exteral DB. - db_host: postgresql - # The password of the Clair's postgres database. Only effective when Harbor is deployed with Clair. - # Please update it before deployment. Subsequent update will cause Clair's API server and Harbor unable to access Clair's database. - db_password: root123 - # Clair DB connect port - db_port: 5432 - # Clair DB username - db_username: postgres - # Clair default database - db: postgres - # The interval of clair updaters, the unit is hour, set to 0 to disable the updaters. - updaters_interval: 12 - - #Config http proxy for Clair, e.g. http://my.proxy.com:3128 - #Clair doesn't need to connect to harbor internal components via http proxy. - http_proxy: - https_proxy: - no_proxy: 127.0.0.1,localhost,core,registry - -# Harbor Storage settings -storage: - #Please be aware that the following storage settings will be applied to both docker registry and helm chart repository. - #registry_storage_provider can be: filesystem, s3, gcs, azure, etc. - registry_storage_provider_name: filesystem - #registry_storage_provider_config is a comma separated "key: value" pairs, e.g. "key1: value, key2: value2". - #To avoid duplicated configurations, both docker registry and chart repository follow the same storage configuration specifications of docker registry. - #Refer to https://docs.docker.com/registry/configuration/#storage for all available configuration. - registry_storage_provider_config: - #registry_custom_ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore - #of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signed certificate. - registry_custom_ca_bundle: - -#If reload_config=true, all settings which present in harbor.yml take effect after prepare and restart harbor, it overwrites exsiting settings. -#reload_config=true -#Regular expression to match skipped environment variables -#skip_reload_env_pattern: (^EMAIL.*)|(^LDAP.*) +# Umcomments external_redis if using external Redis server +# external_redis: +# host: redis +# port: 6379 +# password: +# # db_index 0 is for core, it's unchangeable +# registry_db_index: 1 +# jobservice_db_index: 2 +# chartmuseum_db_index: 3 diff --git a/make/install.sh b/make/install.sh index 25247c951..b01da719c 100755 --- a/make/install.sh +++ b/make/install.sh @@ -192,9 +192,9 @@ docker-compose up -d protocol=http hostname=reg.mydomain.com -if [[ $(cat ./harbor.yml) =~ ui_url_protocol:[[:blank:]]*(https?) ]] +if [ -n "$(grep '^[^#]*https:' ./harbor.yml)"] then -protocol=${BASH_REMATCH[1]} +protocol=https fi if [[ $(grep '^[[:blank:]]*hostname:' ./harbor.yml) =~ hostname:[[:blank:]]*(.*) ]] diff --git a/make/photon/prepare/g.py b/make/photon/prepare/g.py index 810816781..bb766f07b 100644 --- a/make/photon/prepare/g.py +++ b/make/photon/prepare/g.py @@ -20,10 +20,8 @@ private_key_pem_path = Path('/secret/core/private_key.pem') root_crt_path = Path('/secret/registry/root.crt') config_file_path = '/compose_location/harbor.yml' +input_config_path = '/input/harbor.yml' versions_file_path = Path('/usr/src/app/versions') cert_dir = os.path.join(config_dir, "nginx", "cert") -core_cert_dir = os.path.join(config_dir, "core", "certificates") - -registry_custom_ca_bundle_storage_path = Path('/secret/common/custom-ca-bundle.crt') -registry_custom_ca_bundle_storage_input_path = Path('/input/common/custom-ca-bundle.crt') \ No newline at end of file +core_cert_dir = os.path.join(config_dir, "core", "certificates") \ No newline at end of file diff --git a/make/photon/prepare/main.py b/make/photon/prepare/main.py index 8ba6c76cd..604d2735c 100644 --- a/make/photon/prepare/main.py +++ b/make/photon/prepare/main.py @@ -4,7 +4,7 @@ import click from utils.misc import delfile from utils.configs import validate, parse_yaml_config -from utils.cert import prepare_ca, SSL_CERT_KEY_PATH, SSL_CERT_PATH, get_secret_key, copy_ssl_cert, copy_secret_keys +from utils.cert import prepare_ca, SSL_CERT_KEY_PATH, SSL_CERT_PATH, get_secret_key from utils.db import prepare_db from utils.jobservice import prepare_job_service from utils.registry import prepare_registry @@ -16,13 +16,12 @@ from utils.clair import prepare_clair from utils.chart import prepare_chartmuseum from utils.docker_compose import prepare_docker_compose from utils.nginx import prepare_nginx, nginx_confd_dir -from g import (config_dir, config_file_path, private_key_pem_path, root_crt_path, -registry_custom_ca_bundle_storage_path, registry_custom_ca_bundle_storage_input_path, secret_key_dir, +from g import (config_dir, input_config_path, private_key_pem_path, root_crt_path, secret_key_dir, old_private_key_pem_path, old_crt_path) # Main function @click.command() -@click.option('--conf', default=config_file_path, help="the path of Harbor configuration file") +@click.option('--conf', default=input_config_path, help="the path of Harbor configuration file") @click.option('--with-notary', is_flag=True, help="the Harbor instance is to be deployed with notary") @click.option('--with-clair', is_flag=True, help="the Harbor instance is to be deployed with clair") @click.option('--with-chartmuseum', is_flag=True, help="the Harbor instance is to be deployed with chart repository supporting") @@ -40,21 +39,14 @@ def main(conf, with_notary, with_clair, with_chartmuseum): prepare_db(config_dict) prepare_job_service(config_dict) - copy_secret_keys() get_secret_key(secret_key_dir) - if config_dict['protocol'] == 'https': - copy_ssl_cert() - # If Customized cert enabled prepare_ca( private_key_pem_path=private_key_pem_path, root_crt_path=root_crt_path, old_private_key_pem_path=old_private_key_pem_path, - old_crt_path=old_crt_path, - registry_custom_ca_bundle_config=registry_custom_ca_bundle_storage_input_path, - registry_custom_ca_bundle_storage_path=registry_custom_ca_bundle_storage_path) - + old_crt_path=old_crt_path) if with_notary: prepare_notary(config_dict, nginx_confd_dir, SSL_CERT_PATH, SSL_CERT_KEY_PATH) diff --git a/make/photon/prepare/templates/core/config_env.jinja b/make/photon/prepare/templates/core/config_env.jinja index c6485ad36..ce986e7d3 100644 --- a/make/photon/prepare/templates/core/config_env.jinja +++ b/make/photon/prepare/templates/core/config_env.jinja @@ -1,16 +1,6 @@ PORT=8080 -LOG_LEVEL=info +LOG_LEVEL={{log_level}} EXT_ENDPOINT={{public_url}} -SELF_REGISTRATION={{self_registration}} -LDAP_URL={{ldap_url}} -LDAP_SEARCH_DN={{ldap_searchdn}} -LDAP_SEARCH_PWD={{ldap_search_pwd}} -LDAP_BASE_DN={{ldap_basedn}} -LDAP_FILTER={{ldap_filter}} -LDAP_UID={{ldap_uid}} -LDAP_SCOPE={{ldap_scope}} -LDAP_TIMEOUT={{ldap_timeout}} -LDAP_VERIFY_CERT={{ldap_verify_cert}} DATABASE_TYPE=postgresql POSTGRESQL_HOST={{db_host}} POSTGRESQL_PORT={{db_port}} @@ -18,50 +8,29 @@ POSTGRESQL_USERNAME={{db_user}} POSTGRESQL_PASSWORD={{db_password}} POSTGRESQL_DATABASE=registry POSTGRESQL_SSLMODE=disable -LDAP_GROUP_BASEDN={{ldap_group_basedn}} -LDAP_GROUP_FILTER={{ldap_group_filter}} -LDAP_GROUP_GID={{ldap_group_gid}} -LDAP_GROUP_SCOPE={{ldap_group_scope}} REGISTRY_URL={{registry_url}} TOKEN_SERVICE_URL={{token_service_url}} -EMAIL_HOST={{email_host}} -EMAIL_PORT={{email_port}} -EMAIL_USR={{email_usr}} -EMAIL_PWD={{email_pwd}} -EMAIL_SSL={{email_ssl}} -EMAIL_FROM={{email_from}} -EMAIL_IDENTITY={{email_identity}} -EMAIL_INSECURE={{email_insecure}} HARBOR_ADMIN_PASSWORD={{harbor_admin_password}} -PROJECT_CREATION_RESTRICTION={{project_creation_restriction}} MAX_JOB_WORKERS={{max_job_workers}} CORE_SECRET={{core_secret}} JOBSERVICE_SECRET={{jobservice_secret}} -TOKEN_EXPIRATION={{token_expiration}} CFG_EXPIRATION=5 ADMIRAL_URL={{admiral_url}} WITH_NOTARY={{with_notary}} WITH_CLAIR={{with_clair}} -CLAIR_DB_PASSWORD={{clair_db_password}} -CLAIR_DB_HOST={{clair_db_host}} -CLAIR_DB_PORT={{clair_db_port}} -CLAIR_DB_USERNAME={{clair_db_username}} +CLAIR_DB_PASSWORD={{db_password}} +CLAIR_DB_HOST={{db_host}} +CLAIR_DB_PORT={{db_port}} +CLAIR_DB_USERNAME={{db_user}} CLAIR_DB={{clair_db}} CLAIR_DB_SSLMODE=disable -RESET={{reload_config}} -UAA_ENDPOINT={{uaa_endpoint}} -UAA_CLIENTID={{uaa_clientid}} -UAA_CLIENTSECRET={{uaa_clientsecret}} -UAA_VERIFY_CERT={{uaa_verify_cert}} CORE_URL={{core_url}} JOBSERVICE_URL={{jobservice_url}} CLAIR_URL={{clair_url}} NOTARY_URL={{notary_url}} REGISTRY_STORAGE_PROVIDER_NAME={{storage_provider_name}} READ_ONLY=false -SKIP_RELOAD_ENV_PATTERN={{skip_reload_env_pattern}} RELOAD_KEY={{reload_key}} CHART_REPOSITORY_URL={{chart_repository_url}} -LDAP_GROUP_ADMIN_DN={{ldap_group_admin_dn}} REGISTRY_CONTROLLER_URL={{registry_controller_url}} WITH_CHARTMUSEUM={{with_chartmuseum}} diff --git a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja index 37a5dafd8..b290a543c 100644 --- a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja +++ b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja @@ -33,8 +33,11 @@ services: - {{data_volume}}/registry:/storage:z - ./common/config/registry/:/etc/registry/:z - {{data_volume}}/secret/registry/root.crt:/etc/registry/root.crt:z -{%if registry_custom_ca_bundle_storage_path %} - - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z +{% if gcs_keyfile %} + - {{gcs_keyfile}}:/etc/registry/gcs.key +{% endif %} +{%if registry_custom_ca_bundle_path %} + - {{registry_custom_ca_bundle_path}}:/harbor_cust_cert/custom-ca-bundle.crt:z {% endif %} networks: - harbor @@ -247,8 +250,8 @@ services: volumes: - ./common/config/nginx:/etc/nginx:z {% if protocol == 'https' %} - - {{data_volume}}/secret/nginx/server.key:/etc/nginx/cert/server.key - - {{data_volume}}/secret/nginx/server.crt:/etc/nginx/cert/server.crt + - {{cert_key_path}}:/etc/nginx/cert/server.key:z + - {{cert_path}}:/etc/nginx/cert/server.crt:z {% endif %} networks: - harbor @@ -257,8 +260,8 @@ services: {% endif %} dns_search: . ports: - - 80:80 - - 443:443 + - {{http_port}}:80 + - {{https_port}}:443 - 4443:4443 depends_on: - postgresql @@ -337,8 +340,8 @@ services: - postgresql volumes: - ./common/config/clair/config.yaml:/etc/clair/config.yaml:z -{%if registry_custom_ca_bundle_storage_path %} - - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z +{%if registry_custom_ca_bundle_path %} + - {{registry_custom_ca_bundle_path}}:/harbor_cust_cert/custom-ca-bundle.crt:z {% endif %} logging: driver: "syslog" @@ -368,8 +371,8 @@ services: volumes: - {{data_volume}}/chart_storage:/chart_storage:z - ./common/config/chartserver:/etc/chartserver:z -{%if registry_custom_ca_bundle_storage_path %} - - {{data_volume}}/secret/common/custom-ca-bundle.crt:/harbor_cust_cert/custom-ca-bundle.crt:z +{%if registry_custom_ca_bundle_path %} + - {{registry_custom_ca_bundle_path}}:/harbor_cust_cert/custom-ca-bundle.crt:z {% endif %} logging: driver: "syslog" diff --git a/make/photon/prepare/templates/registry/config.yml.jinja b/make/photon/prepare/templates/registry/config.yml.jinja index 9d649d565..8af33b2d3 100644 --- a/make/photon/prepare/templates/registry/config.yml.jinja +++ b/make/photon/prepare/templates/registry/config.yml.jinja @@ -1,6 +1,6 @@ version: 0.1 log: - level: info + level: {{log_level}} fields: service: registry storage: diff --git a/make/photon/prepare/utils/cert.py b/make/photon/prepare/utils/cert.py index 260a2b574..0c75f615c 100644 --- a/make/photon/prepare/utils/cert.py +++ b/make/photon/prepare/utils/cert.py @@ -10,16 +10,11 @@ from .misc import generate_random_string SSL_CERT_PATH = os.path.join("/etc/nginx/cert", "server.crt") SSL_CERT_KEY_PATH = os.path.join("/etc/nginx/cert", "server.key") -input_cert = '/input/nginx/server.crt' -input_cert_key = '/input/nginx/server.key' - secret_cert_dir = '/secret/nginx' secret_cert = '/secret/nginx/server.crt' secret_cert_key = '/secret/nginx/server.key' -input_secret_keys_dir = '/input/keys' secret_keys_dir = '/secret/keys' -allowed_secret_key_names = ['defaultalias', 'secretkey'] def _get_secret(folder, filename, length=16): key_file = os.path.join(folder, filename) @@ -50,26 +45,6 @@ def get_alias(path): alias = _get_secret(path, "defaultalias", length=8) return alias -def copy_secret_keys(): - """ - Copy the secret keys, which used for encrypt user password, from input keys dir to secret keys dir - """ - if os.path.isdir(input_secret_keys_dir) and os.path.isdir(secret_keys_dir): - input_files = os.listdir(input_secret_keys_dir) - secret_files = os.listdir(secret_keys_dir) - files_need_copy = [x for x in input_files if (x in allowed_secret_key_names) and (x not in secret_files) ] - for f in files_need_copy: - shutil.copy(f, secret_keys_dir) - -def copy_ssl_cert(): - """ - Copy the ssl certs key paris, which used in nginx ssl certificate, from input dir to secret cert dir - """ - if os.path.isfile(input_cert_key) and os.path.isfile(input_cert): - os.makedirs(secret_cert_dir, exist_ok=True) - shutil.copy(input_cert, secret_cert) - shutil.copy(input_cert_key, secret_cert_key) - ## decorator actions def stat_decorator(func): @wraps(func) @@ -115,9 +90,7 @@ def prepare_ca( private_key_pem_path: Path, root_crt_path: Path, old_private_key_pem_path: Path, - old_crt_path: Path, - registry_custom_ca_bundle_config: Path, - registry_custom_ca_bundle_storage_path: Path): + old_crt_path: Path): if not ( private_key_pem_path.exists() and root_crt_path.exists() ): # From version 1.8 the cert storage path is changed # if old key paris not exist create new ones @@ -132,11 +105,4 @@ def prepare_ca( mark_file(root_crt_path) else: shutil.move(old_crt_path, root_crt_path) - shutil.move(old_private_key_pem_path, private_key_pem_path) - - - if not registry_custom_ca_bundle_storage_path.exists() and registry_custom_ca_bundle_config.exists(): - registry_custom_ca_bundle_storage_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(registry_custom_ca_bundle_config, registry_custom_ca_bundle_storage_path) - mark_file(registry_custom_ca_bundle_storage_path) - print("Copied custom ca bundle: %s" % registry_custom_ca_bundle_config) \ No newline at end of file + shutil.move(old_private_key_pem_path, private_key_pem_path) \ No newline at end of file diff --git a/make/photon/prepare/utils/chart.py b/make/photon/prepare/utils/chart.py index 825c80feb..68bbae87e 100644 --- a/make/photon/prepare/utils/chart.py +++ b/make/photon/prepare/utils/chart.py @@ -24,11 +24,6 @@ def prepare_chartmuseum(config_dict): print ("Create config folder: %s" % chartm_config_dir) os.makedirs(chartm_config_dir) - # handle custom ca bundle - if len(registry_custom_ca_bundle_path) > 0 and os.path.isfile(registry_custom_ca_bundle_path): - shutil.copyfile(registry_custom_ca_bundle_path, os.path.join(chartm_config_dir, "custom-ca-bundle.crt")) - print("Copied custom ca bundle: %s" % os.path.join(chartm_config_dir, "custom-ca-bundle.crt")) - # process redis info cache_store = "redis" cache_redis_password = redis_password @@ -42,18 +37,9 @@ def prepare_chartmuseum(config_dict): # storage provider configurations # please be aware that, we do not check the validations of the values for the specified keys # convert the configs to config map - storage_provider_configs = storage_provider_config.split(",") - storgae_provider_confg_map = {} + storgae_provider_confg_map = storage_provider_config storage_provider_config_options = [] - for k_v in storage_provider_configs: - if len(k_v) > 0: - kvs = k_v.split(": ") # add space suffix to avoid existing ":" in the value - if len(kvs) == 2: - #key must not be empty - if kvs[0].strip() != "": - storgae_provider_confg_map[kvs[0].strip()] = kvs[1].strip() - if storage_provider_name == "s3": # aws s3 storage storage_driver = "amazon" diff --git a/make/photon/prepare/utils/clair.py b/make/photon/prepare/utils/clair.py index 3a3197aee..86f49abd3 100644 --- a/make/photon/prepare/utils/clair.py +++ b/make/photon/prepare/utils/clair.py @@ -27,17 +27,17 @@ def prepare_clair(config_dict): render_jinja( postgres_env_template, postgres_env_path, - password=config_dict['clair_db_password']) + password=config_dict['db_password']) render_jinja( clair_config_template, clair_config_path, uid=DEFAULT_UID, gid=DEFAULT_GID, - password= config_dict['clair_db_password'], - username= config_dict['clair_db_username'], - host= config_dict['clair_db_host'], - port= config_dict['clair_db_port'], + password= config_dict['db_password'], + username= config_dict['db_user'], + host= config_dict['db_host'], + port= config_dict['db_port'], dbname= config_dict['clair_db'], interval= config_dict['clair_updaters_interval']) diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index 9a7c402ee..1a3b56714 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -37,10 +37,6 @@ def validate(conf, **kwargs): raise Exception( "Error: redis_port in harbor.cfg needs to point to the port of Redis server or cluster.") - redis_db_index = conf.get("redis_db_index") - if len(redis_db_index.split(",")) != 3: - raise Exception( - "Error invalid value for redis_db_index: %s. please set it as 1,2,3" % redis_db_index) def parse_versions(): if not versions_file_path.is_file(): @@ -58,137 +54,127 @@ def parse_yaml_config(config_file_path): with open(config_file_path) as f: configs = yaml.load(f) - config_dict = {} - config_dict['adminserver_url'] = "http://adminserver:8080" - config_dict['registry_url'] = "http://registry:5000" - config_dict['registry_controller_url'] = "http://registryctl:8080" - config_dict['core_url'] = "http://core:8080" - config_dict['token_service_url'] = "http://core:8080/service/token" + config_dict = { + 'adminserver_url': "http://adminserver:8080", + 'registry_url': "http://registry:5000", + 'registry_controller_url': "http://registryctl:8080", + 'core_url': "http://core:8080", + 'token_service_url': "http://core:8080/service/token", + 'jobservice_url': 'http://jobservice:8080', + 'clair_url': 'http://clair:6060', + 'notary_url': 'http://notary-server:4443', + 'chart_repository_url': 'http://chartmuseum:9999' + } - config_dict['jobservice_url'] = "http://jobservice:8080" - config_dict['clair_url'] = "http://clair:6060" - config_dict['notary_url'] = "http://notary-server:4443" - config_dict['chart_repository_url'] = "http://chartmuseum:9999" + config_dict['hostname'] = configs["hostname"] - if configs.get("reload_config"): - config_dict['reload_config'] = configs.get("reload_config") - else: - config_dict['reload_config'] = "false" + config_dict['protocol'] = 'http' + http_config = configs.get('http') or {} + config_dict['http_port'] = http_config.get('port', 80) - config_dict['hostname'] = configs.get("hostname") - config_dict['protocol'] = configs.get("ui_url_protocol") - config_dict['public_url'] = config_dict['protocol'] + "://" + config_dict['hostname'] + https_config = configs.get('https') + if https_config: + config_dict['protocol'] = 'https' + config_dict['https_port'] = https_config.get('port', 443) + config_dict['cert_path'] = https_config["certificate"] + config_dict['cert_key_path'] = https_config["private_key"] - # Data path volume - config_dict['data_volume'] = configs.get("data_volume") - - # Email related configs - config_dict['email_identity'] = configs.get("email_identity") - config_dict['email_host'] = configs.get("email_server") - config_dict['email_port'] = configs.get("email_server_port") - config_dict['email_usr'] = configs.get("email_username") - config_dict['email_pwd'] = configs.get("email_password") - config_dict['email_from'] = configs.get("email_from") - config_dict['email_ssl'] = configs.get("email_ssl") - config_dict['email_insecure'] = configs.get("email_insecure") - config_dict['harbor_admin_password'] = configs.get("harbor_admin_password") - config_dict['auth_mode'] = configs.get("auth_mode") - config_dict['ldap_url'] = configs.get("ldap_url") - - # LDAP related configs - # this two options are either both set or unset - if configs.get("ldap_searchdn"): - config_dict['ldap_searchdn'] = configs["ldap_searchdn"] - config_dict['ldap_search_pwd'] = configs["ldap_search_pwd"] - else: - config_dict['ldap_searchdn'] = "" - config_dict['ldap_search_pwd'] = "" - config_dict['ldap_basedn'] = configs.get("ldap_basedn") - # ldap_filter is null by default - if configs.get("ldap_filter"): - config_dict['ldap_filter'] = configs["ldap_filter"] - else: - config_dict['ldap_filter'] = "" - config_dict['ldap_uid'] = configs.get("ldap_uid") - config_dict['ldap_scope'] = configs.get("ldap_scope") - config_dict['ldap_timeout'] = configs.get("ldap_timeout") - config_dict['ldap_verify_cert'] = configs.get("ldap_verify_cert") - config_dict['ldap_group_basedn'] = configs.get("ldap_group_basedn") - config_dict['ldap_group_filter'] = configs.get("ldap_group_filter") - config_dict['ldap_group_gid'] = configs.get("ldap_group_gid") - config_dict['ldap_group_scope'] = configs.get("ldap_group_scope") - # Admin dn - config_dict['ldap_group_admin_dn'] = configs.get("ldap_group_admin_dn") or '' + config_dict['public_url'] = configs.get('external_url') or '{protocol}://{hostname}'.format(**config_dict) # DB configs db_configs = configs.get('database') - config_dict['db_host'] = db_configs.get("host") - config_dict['db_port'] = db_configs.get("port") - config_dict['db_user'] = db_configs.get("username") - config_dict['db_password'] = db_configs.get("password") + if db_configs: + config_dict['db_host'] = 'postgresql' + config_dict['db_port'] = 5432 + config_dict['db_user'] = 'postgres' + config_dict['db_password'] = db_configs.get("password") or '' + config_dict['ssl_mode'] = 'disable' - config_dict['self_registration'] = configs.get("self_registration") - config_dict['project_creation_restriction'] = configs.get("project_creation_restriction") - # secure configs - if config_dict['protocol'] == "https": - config_dict['cert_path'] = configs.get("ssl_cert") - config_dict['cert_key_path'] = configs.get("ssl_cert_key") - config_dict['customize_crt'] = configs.get("customize_crt") - config_dict['max_job_workers'] = configs.get("max_job_workers") - config_dict['token_expiration'] = configs.get("token_expiration") + # Data path volume + config_dict['data_volume'] = configs['data_volume'] - config_dict['secretkey_path'] = configs["secretkey_path"] - # Admiral configs - if configs.get("admiral_url"): - config_dict['admiral_url'] = configs["admiral_url"] + # Initial Admin Password + config_dict['harbor_admin_password'] = configs["harbor_admin_password"] + + # Registry storage configs + storage_config = configs.get('storage_service') or {} + + config_dict['registry_custom_ca_bundle_path'] = storage_config.get('ca_bundle') or '' + + if storage_config.get('filesystem'): + config_dict['storage_provider_name'] = 'filesystem' + config_dict['storage_provider_config'] = storage_config['filesystem'] + elif storage_config.get('azure'): + config_dict['storage_provider_name'] = 'azure' + config_dict['storage_provider_config'] = storage_config['azure'] + elif storage_config.get('gcs'): + config_dict['storage_provider_name'] = 'gcs' + config_dict['storage_provider_config'] = storage_config['gcs'] + elif storage_config.get('s3'): + config_dict['storage_provider_name'] = 's3' + config_dict['storage_provider_config'] = storage_config['s3'] + elif storage_config.get('swift'): + config_dict['storage_provider_name'] = 'swift' + config_dict['storage_provider_config'] = storage_config['swift'] + elif storage_config.get('oss'): + config_dict['storage_provider_name'] = 'oss' + config_dict['storage_provider_config'] = storage_config['oss'] else: - config_dict['admiral_url'] = "" + config_dict['storage_provider_name'] = 'filesystem' + config_dict['storage_provider_config'] = {} # Clair configs clair_configs = configs.get("clair") or {} - config_dict['clair_db_password'] = clair_configs.get("db_password") or '' - config_dict['clair_db_host'] = clair_configs.get("db_host") or '' - config_dict['clair_db_port'] = clair_configs.get("db_port") or '' - config_dict['clair_db_username'] = clair_configs.get("db_username") or '' - config_dict['clair_db'] = clair_configs.get("db") or '' - config_dict['clair_updaters_interval'] = clair_configs.get("updaters_interval") or '' + config_dict['clair_db'] = 'postgres' + config_dict['clair_updaters_interval'] = clair_configs.get("updaters_interval") or 12 config_dict['clair_http_proxy'] = clair_configs.get('http_proxy') or '' config_dict['clair_https_proxy'] = clair_configs.get('https_proxy') or '' - config_dict['clair_no_proxy'] = clair_configs.get('no_proxy') or '' + config_dict['clair_no_proxy'] = clair_configs.get('no_proxy') or '127.0.0.1,localhost,core,registry' + + # jobservice config + js_config = configs.get('jobservice') or {} + config_dict['max_job_workers'] = js_config["max_job_workers"] + config_dict['jobservice_secret'] = generate_random_string(16) - # UAA configs - config_dict['uaa_endpoint'] = configs.get("uaa_endpoint") - config_dict['uaa_clientid'] = configs.get("uaa_clientid") - config_dict['uaa_clientsecret'] = configs.get("uaa_clientsecret") - config_dict['uaa_verify_cert'] = configs.get("uaa_verify_cert") - config_dict['uaa_ca_cert'] = configs.get("uaa_ca_cert") # Log configs log_configs = configs.get('log') or {} - config_dict['log_location'] = log_configs.get("location") - config_dict['log_rotate_count'] = log_configs.get("rotate_count") - config_dict['log_rotate_size'] = log_configs.get("rotate_size") + config_dict['log_location'] = log_configs["location"] + config_dict['log_rotate_count'] = log_configs["rotate_count"] + config_dict['log_rotate_size'] = log_configs["rotate_size"] + config_dict['log_level'] = log_configs['level'] - # Redis configs - redis_configs = configs.get("redis") + + # external DB, if external_db enabled, it will cover the database config + external_db_configs = configs.get('external_database') or {} + if external_db_configs: + config_dict['db_password'] = external_db_configs.get('password') or '' + config_dict['db_host'] = external_db_configs['host'] + config_dict['db_port'] = external_db_configs['port'] + config_dict['db_user'] = db_configs['username'] + if external_db_configs.get('ssl_mode'): + config_dict['db_ssl_mode'] = external_db_configs['ssl_mode'] + + + # redis config + redis_configs = configs.get("external_redis") if redis_configs: - config_dict['redis_host'] = redis_configs.get("host") or '' - config_dict['redis_port'] = redis_configs.get("port") or '' + # using external_redis + config_dict['redis_host'] = redis_configs['host'] + config_dict['redis_port'] = redis_configs['port'] config_dict['redis_password'] = redis_configs.get("password") or '' - config_dict['redis_db_index'] = redis_configs.get("db_index") or '' - db_indexs = config_dict['redis_db_index'].split(',') - config_dict['redis_db_index_reg'] = db_indexs[0] - config_dict['redis_db_index_js'] = db_indexs[1] - config_dict['redis_db_index_chart'] = db_indexs[2] + config_dict['redis_db_index_reg'] = redis_configs.get('registry_db_index') or 1 + config_dict['redis_db_index_js'] = redis_configs.get('jobservice_db_index') or 2 + config_dict['redis_db_index_chart'] = redis_configs.get('chartmuseum_db_index') or 3 else: - config_dict['redis_host'] = '' - config_dict['redis_port'] = '' + ## Using local redis + config_dict['redis_host'] = 'redis' + config_dict['redis_port'] = 6379 config_dict['redis_password'] = '' - config_dict['redis_db_index'] = '' - config_dict['redis_db_index_reg'] = '' - config_dict['redis_db_index_js'] = '' - config_dict['redis_db_index_chart'] = '' + config_dict['redis_db_index_reg'] = 1 + config_dict['redis_db_index_js'] = 2 + config_dict['redis_db_index_chart'] = 3 # redis://[arbitrary_username:password@]ipaddress:port/database_index if config_dict.get('redis_password'): @@ -198,26 +184,10 @@ def parse_yaml_config(config_file_path): config_dict['redis_url_js'] = "redis://%s:%s/%s" % (config_dict['redis_host'], config_dict['redis_port'], config_dict['redis_db_index_js']) config_dict['redis_url_reg'] = "redis://%s:%s/%s" % (config_dict['redis_host'], config_dict['redis_port'], config_dict['redis_db_index_reg']) - if configs.get("skip_reload_env_pattern"): - config_dict['skip_reload_env_pattern'] = configs["skip_reload_env_pattern"] - else: - config_dict['skip_reload_env_pattern'] = "$^" - - # Registry storage configs - storage_config = configs.get('storage') - if storage_config: - config_dict['storage_provider_name'] = storage_config.get("registry_storage_provider_name") or '' - config_dict['storage_provider_config'] = storage_config.get("registry_storage_provider_config") or '' - # yaml requires 1 or more spaces between the key and value - config_dict['storage_provider_config'] = config_dict['storage_provider_config'].replace(":", ": ", 1) - config_dict['registry_custom_ca_bundle_path'] = storage_config.get("registry_custom_ca_bundle") or '' - else: - config_dict['storage_provider_name'] = '' - config_dict['storage_provider_config'] = '' - config_dict['registry_custom_ca_bundle_path'] = '' - - # auto generate secret string + # auto generated secret string for core config_dict['core_secret'] = generate_random_string(16) - config_dict['jobservice_secret'] = generate_random_string(16) + + # Admiral configs + config_dict['admiral_url'] = configs.get("admiral_url") or "" return config_dict \ No newline at end of file diff --git a/make/photon/prepare/utils/core.py b/make/photon/prepare/utils/core.py index ecf9edfd8..8d04f5b1c 100644 --- a/make/photon/prepare/utils/core.py +++ b/make/photon/prepare/utils/core.py @@ -43,7 +43,6 @@ def copy_core_config(core_templates_path, core_config_path): def render_config_env(config_dict, with_notary, with_clair, with_chartmuseum): # Use reload_key to avoid reload config after restart harbor - reload_key = generate_random_string(6) if config_dict['reload_config'] == "true" else "" render_jinja( core_config_env_template, @@ -51,6 +50,5 @@ def render_config_env(config_dict, with_notary, with_clair, with_chartmuseum): with_notary=with_notary, with_clair=with_clair, with_chartmuseum=with_chartmuseum, - reload_key=reload_key, **config_dict ) \ No newline at end of file diff --git a/make/photon/prepare/utils/docker_compose.py b/make/photon/prepare/utils/docker_compose.py index 28ece71cd..8027e36c2 100644 --- a/make/photon/prepare/utils/docker_compose.py +++ b/make/photon/prepare/utils/docker_compose.py @@ -28,10 +28,17 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum): 'cert_key_path': configs['cert_key_path'], 'cert_path': configs['cert_path'], 'protocol': configs['protocol'], - 'registry_custom_ca_bundle_storage_path': configs['registry_custom_ca_bundle_path'], + 'http_port': configs['http_port'], + 'registry_custom_ca_bundle_path': configs['registry_custom_ca_bundle_path'], 'with_notary': with_notary, 'with_clair': with_clair, 'with_chartmuseum': with_chartmuseum } + storage_config = configs.get('storage_provider_config') or {} + if storage_config.get('keyfile'): + rendering_variables['gcs_keyfile'] = storage_config['keyfile'] + if configs.get('https_port'): + rendering_variables['https_port'] = configs['https_port'] + render_jinja(docker_compose_template_path, docker_compose_yml_path, **rendering_variables) \ No newline at end of file diff --git a/make/photon/prepare/utils/misc.py b/make/photon/prepare/utils/misc.py index 1e1f05db1..fe6bcc7f8 100644 --- a/make/photon/prepare/utils/misc.py +++ b/make/photon/prepare/utils/misc.py @@ -36,13 +36,6 @@ def validate(conf, **kwargs): raise Exception( "Error: The path for certificate key: %s is invalid" % cert_key_path) - # Project validate - project_creation = conf.get( - "configuration", "project_creation_restriction") - if project_creation != "everyone" and project_creation != "adminonly": - raise Exception( - "Error invalid value for project_creation_restriction: %s" % project_creation) - # Storage validate valid_storage_drivers = ["filesystem", "azure", "gcs", "s3", "swift", "oss"] diff --git a/make/photon/prepare/utils/nginx.py b/make/photon/prepare/utils/nginx.py index 0eec3febd..08903263a 100644 --- a/make/photon/prepare/utils/nginx.py +++ b/make/photon/prepare/utils/nginx.py @@ -22,11 +22,13 @@ def prepare_nginx(config_dict): def render_nginx_template(config_dict): if config_dict['protocol'] == "https": render_jinja(nginx_https_conf_template, nginx_conf, - ssl_cert = SSL_CERT_PATH, - ssl_cert_key = SSL_CERT_KEY_PATH) + ssl_cert=SSL_CERT_PATH, + ssl_cert_key=SSL_CERT_KEY_PATH) location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTPS else: - render_jinja(nginx_http_conf_template, nginx_conf) + render_jinja( + nginx_http_conf_template, + nginx_conf) location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTP copy_nginx_location_configs_if_exist(nginx_template_ext_dir, nginx_confd_dir, location_file_pattern) diff --git a/make/photon/prepare/utils/notary.py b/make/photon/prepare/utils/notary.py index 041d46be1..db791ea64 100644 --- a/make/photon/prepare/utils/notary.py +++ b/make/photon/prepare/utils/notary.py @@ -19,7 +19,7 @@ notary_signer_env_path = os.path.join(notary_config_dir, "signer_env") notary_server_env_path = os.path.join(notary_config_dir, "server_env") -def prepare_env_notary(customize_crt, nginx_config_dir): +def prepare_env_notary(nginx_config_dir): notary_config_dir = prepare_config_dir(config_dir, "notary") old_signer_cert_secret_path = pathlib.Path(os.path.join(config_dir, 'notary-signer.crt')) old_signer_key_secret_path = pathlib.Path(os.path.join(config_dir, 'notary-signer.key')) @@ -87,7 +87,7 @@ def prepare_env_notary(customize_crt, nginx_config_dir): def prepare_notary(config_dict, nginx_config_dir, ssl_cert_path, ssl_cert_key_path): - prepare_env_notary(config_dict['customize_crt'], nginx_config_dir) + prepare_env_notary(nginx_config_dir) render_jinja( notary_signer_pg_template, diff --git a/make/photon/prepare/utils/registry.py b/make/photon/prepare/utils/registry.py index a319465c9..c8a32463a 100644 --- a/make/photon/prepare/utils/registry.py +++ b/make/photon/prepare/utils/registry.py @@ -1,4 +1,4 @@ -import os, shutil +import os, copy from g import config_dir, templates_dir, DEFAULT_GID, DEFAULT_UID from utils.misc import prepare_config_dir @@ -11,12 +11,11 @@ registry_conf = os.path.join(config_dir, "registry", "config.yml") def prepare_registry(config_dict): - prepare_registry_config_dir() + prepare_config_dir(registry_config_dir) storage_provider_info = get_storage_provider_info( config_dict['storage_provider_name'], - config_dict['storage_provider_config'], - registry_config_dir) + config_dict['storage_provider_config']) render_jinja( registry_config_template_path, @@ -26,26 +25,17 @@ def prepare_registry(config_dict): storage_provider_info=storage_provider_info, **config_dict) -def prepare_registry_config_dir(): - prepare_config_dir(registry_config_dir) -def get_storage_provider_info(provider_name, provider_config, registry_config_dir_path): +def get_storage_provider_info(provider_name, provider_config): + provider_config_copy = copy.deepcopy(provider_config) if provider_name == "filesystem": - if not provider_config: - storage_provider_config = "rootdirectory: /storage" - elif "rootdirectory:" not in storage_provider_config: - storage_provider_config = "rootdirectory: /storage" + "," + storage_provider_config + if not (provider_config_copy and provider_config_copy.has_key('rootdirectory')): + provider_config_copy['rootdirectory'] = '/storage' + if provider_name == 'gcs' and provider_config_copy.get('keyfile'): + provider_config_copy['keyfile'] = '/etc/registry/gcs.key' # generate storage configuration section in yaml format storage_provider_conf_list = [provider_name + ':'] - for c in storage_provider_config.split(","): - kvs = c.split(": ") - if len(kvs) == 2: - if kvs[0].strip() == "keyfile": - srcKeyFile = kvs[1].strip() - if os.path.isfile(srcKeyFile): - shutil.copyfile(srcKeyFile, os.path.join(registry_config_dir_path, "gcs.key")) - storage_provider_conf_list.append("keyfile: %s" % "/etc/registry/gcs.key") - continue - storage_provider_conf_list.append(c.strip()) + for config in provider_config_copy.items(): + storage_provider_conf_list.append('{}: {}'.format(*config)) storage_provider_info = ('\n' + ' ' * 4).join(storage_provider_conf_list) return storage_provider_info diff --git a/make/photon/prepare/utils/uaa.py b/make/photon/prepare/utils/uaa.py deleted file mode 100644 index 151bba54f..000000000 --- a/make/photon/prepare/utils/uaa.py +++ /dev/null @@ -1,11 +0,0 @@ -import os, shutil - -def prepare_uaa_cert_file(uaa_ca_cert, core_cert_dir): - if os.path.isfile(uaa_ca_cert): - if not os.path.isdir(core_cert_dir): - os.makedirs(core_cert_dir) - core_uaa_ca = os.path.join(core_cert_dir, "uaa_ca.pem") - print("Copying UAA CA cert to %s" % core_uaa_ca) - shutil.copyfile(uaa_ca_cert, core_uaa_ca) - else: - print("Can not find UAA CA cert: %s, skip" % uaa_ca_cert) \ No newline at end of file diff --git a/make/prepare b/make/prepare index 5753bb637..a2871b659 100755 --- a/make/prepare +++ b/make/prepare @@ -1,39 +1,40 @@ #!/bin/bash +set +e # If compling source code this dir is harbor's make dir # If install harbor via pacakge, this dir is harbor's root dir harbor_prepare_path="$( cd "$(dirname "$0")" ; pwd -P )" - -echo host make path is set to ${harbor_prepare_path} +echo "prepare base dir is set to ${harbor_prepare_path}" data_path=$(grep '^[^#]*data_volume:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') -log_path=$(grep '^[^#]*location:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') -secretkey_path=$(grep '^[^#]*secretkey_path:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') -ssl_cert_path=$(grep '^[^#]*ssl_cert:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') -ssl_cert_key_path=$(grep '^[^#]*ssl_cert_key:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') -registry_custom_ca_bundle=$(grep '^[^#]*registry_custom_ca_bundle:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}') +# If previous secretkeys exist, move it to new location +previous_secretkey_path=/data/secretkey +previous_defaultalias_path=/data/defaultalias + +if [ -f $previous_secretkey_path ]; then + mkdir -p $data_path/secret/keys + mv $previous_secretkey_path $data_path/secret/keys +fi +if [ -f $previous_defaultalias_path ]; then + mkdir -p $data_path/secret/keys + mv $previous_defaultalias_path $data_path/secret/keys +fi + +# Clean up input dir +rm -rf ${harbor_prepare_path}/input # Create a input dirs mkdir -p ${harbor_prepare_path}/input input_dir=${harbor_prepare_path}/input -mkdir -p $input_dir/nginx -mkdir -p $input_dir/keys -mkdir -p $input_dir/common -# Copy nginx config file to input dir -cp $ssl_cert_path $input_dir/nginx/server.crt -cp $ssl_cert_key_path $input_dir/nginx/server.key - -# Copy secretkey to input dir -cp -r $secretkey_path $input_dir/keys - -# Copy ca bundle to input dir -if [ -f $registry_custom_ca_bundle ] -then - cp -r $registry_custom_ca_bundle $input_dir/common/custom-ca-bundle.crt -fi +set -e # Copy harbor.yml to input dir -cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml +if [[ ! "$1" =~ ^\-\- ]] && [ -f "$1" ] +then + cp $1 $input_dir/harbor.yml +else + cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml +fi # Create secret dir secret_dir=${data_path}/secret @@ -44,8 +45,8 @@ docker run -it --rm -v $input_dir:/input \ -v $harbor_prepare_path:/compose_location \ -v $config_dir:/config \ -v $secret_dir:/secret \ - -v $log_path:/var/log/harbor \ goharbor/prepare:dev $@ +echo "Clean up the input dir" # Clean up input dir -rm -rf ${harbor_prepare_path}/input \ No newline at end of file +rm -rf ${harbor_prepare_path}/input diff --git a/src/common/config/store/driver/db.go b/src/common/config/store/driver/db.go index 0711bd4cc..18fe703e0 100644 --- a/src/common/config/store/driver/db.go +++ b/src/common/config/store/driver/db.go @@ -15,13 +15,14 @@ package driver import ( + "os" + "github.com/goharbor/harbor/src/common/config/encrypt" "github.com/goharbor/harbor/src/common/config/metadata" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/common/utils/log" - "os" ) // Database - Used to load/save configuration in database @@ -63,7 +64,7 @@ func (d *Database) Save(cfgs map[string]interface{}) error { for key, value := range cfgs { if item, ok := metadata.Instance().GetByName(key); ok { if os.Getenv("UTTEST") != "true" && item.Scope == metadata.SystemScope { - log.Errorf("system setting can not updated, key %v", key) + // skip to save system setting to db continue } strValue := utils.GetStrValueOfAnyType(value) diff --git a/src/portal/src/app/account/sign-in/sign-in.component.html b/src/portal/src/app/account/sign-in/sign-in.component.html index 2342192e8..0e1a2de49 100644 --- a/src/portal/src/app/account/sign-in/sign-in.component.html +++ b/src/portal/src/app/account/sign-in/sign-in.component.html @@ -3,7 +3,7 @@
- + diff --git a/src/portal/src/app/account/sign-in/sign-in.component.scss b/src/portal/src/app/account/sign-in/sign-in.component.scss index 87fc25d66..bf5800a69 100644 --- a/src/portal/src/app/account/sign-in/sign-in.component.scss +++ b/src/portal/src/app/account/sign-in/sign-in.component.scss @@ -60,6 +60,6 @@ background:transparent; } } -.title{ - margin-bottom: 50px; +.login-oidc{ + margin-top: 50px; } \ No newline at end of file diff --git a/src/portal/src/app/account/sign-in/sign-in.component.ts b/src/portal/src/app/account/sign-in/sign-in.component.ts index 920aabb7e..bd69bd778 100644 --- a/src/portal/src/app/account/sign-in/sign-in.component.ts +++ b/src/portal/src/app/account/sign-in/sign-in.component.ts @@ -141,11 +141,12 @@ export class SignInComponent implements AfterViewChecked, OnInit { return this.appConfig.auth_mode === 'db_auth' && this.appConfig.self_registration; } - + public get isOidcLoginMode(): boolean { + return this.appConfig.auth_mode === 'oidc_auth'; + } public get showForgetPwd(): boolean { return this.appConfig.auth_mode !== 'ldap_auth' && this.appConfig.auth_mode !== 'uaa_auth'; } - clickRememberMe($event: any): void { if ($event && $event.target) { this.rememberMe = $event.target.checked; diff --git a/src/portal/src/app/base/navigator/navigator.component.ts b/src/portal/src/app/base/navigator/navigator.component.ts index 399627e2e..c03a815dd 100644 --- a/src/portal/src/app/base/navigator/navigator.component.ts +++ b/src/portal/src/app/base/navigator/navigator.component.ts @@ -14,7 +14,8 @@ import { Component, Output, EventEmitter, OnInit } from '@angular/core'; import { Router, NavigationExtras } from '@angular/router'; import { TranslateService } from '@ngx-translate/core'; - +// import { map } from 'rxjs/operators'; +import { PlatformLocation } from '@angular/common'; import { ModalEvent } from '../modal-event'; import { modalEvents } from '../modal-events.const'; @@ -25,7 +26,7 @@ import { supportedLangs, enLang, languageNames, CommonRoutes } from '../../share import { AppConfigService } from '../../app-config.service'; import { SearchTriggerService } from '../global-search/search-trigger.service'; import { MessageHandlerService } from '../../shared/message-handler/message-handler.service'; -import {SkinableConfig} from "../../skinable-config.service"; +import { SkinableConfig } from "../../skinable-config.service"; @Component({ selector: 'navigator', @@ -40,12 +41,13 @@ export class NavigatorComponent implements OnInit { selectedLang: string = enLang; appTitle: string = 'APP_TITLE.HARBOR'; - customStyle: {[key: string]: any}; - customProjectName: {[key: string]: any}; + customStyle: { [key: string]: any }; + customProjectName: { [key: string]: any }; constructor( private session: SessionService, private router: Router, + private location: PlatformLocation, private translate: TranslateService, private cookie: CookieService, private appConfigService: AppConfigService, @@ -65,10 +67,10 @@ export class NavigatorComponent implements OnInit { } this.selectedLang = this.translate.currentLang; - this.translate.onLangChange.subscribe((langChange: {lang: string}) => { + this.translate.onLangChange.subscribe((langChange: { lang: string }) => { this.selectedLang = langChange.lang; // Keep in cookie for next use - let opt: CookieOptions = {path: '/', expires: new Date(Date.now() + 3600 * 1000 * 24 * 31)}; + let opt: CookieOptions = { path: '/', expires: new Date(Date.now() + 3600 * 1000 * 24 * 31) }; this.cookie.put("harbor-lang", langChange.lang, opt); }); if (this.appConfigService.isIntegrationMode()) { @@ -112,7 +114,7 @@ export class NavigatorComponent implements OnInit { let config = this.appConfigService.getConfig(); return user && ((config && !(config.auth_mode === "ldap_auth" || config.auth_mode === "uaa_auth")) || - (user.user_id === 1 && user.username === "admin")); + (user.user_id === 1 && user.username === "admin")); } matchLang(lang: string): boolean { @@ -147,8 +149,10 @@ export class NavigatorComponent implements OnInit { logOut(): void { // Naviagte to the sign in route // Appending 'signout' means destroy session cache + let signout = true; + let redirect_url = this.location.pathname; let navigatorExtra: NavigationExtras = { - queryParams: { "signout": true } + queryParams: {signout, redirect_url} }; this.router.navigate([CommonRoutes.EMBEDDED_SIGN_IN], navigatorExtra); // Confirm search result panel is close diff --git a/src/portal/src/app/config/auth/config-auth.component.ts b/src/portal/src/app/config/auth/config-auth.component.ts index 92c23ecb3..9ea4bc1f1 100644 --- a/src/portal/src/app/config/auth/config-auth.component.ts +++ b/src/portal/src/app/config/auth/config-auth.component.ts @@ -149,6 +149,7 @@ export class ConfigurationAuthComponent implements OnChanges { for (let prop in allChanges) { if (prop.startsWith('ldap_') || prop.startsWith('uaa_') + || prop.startsWith('oidc_') || prop === 'auth_mode' || prop === 'project_creattion_restriction' || prop === 'self_registration' diff --git a/src/portal/src/app/oidc-onboard/oidc-onboard.component.ts b/src/portal/src/app/oidc-onboard/oidc-onboard.component.ts index 8b32a4297..beae8b6a8 100644 --- a/src/portal/src/app/oidc-onboard/oidc-onboard.component.ts +++ b/src/portal/src/app/oidc-onboard/oidc-onboard.component.ts @@ -28,7 +28,9 @@ export class OidcOnboardComponent implements OnInit { }); } clickSaveBtn(): void { - this.oidcOnboardService.oidcSave({ username: this.oidcUsername.value }).subscribe(res => { } + this.oidcOnboardService.oidcSave({ username: this.oidcUsername.value }).subscribe(res => { + this.router.navigate([CommonRoutes.HARBOR_DEFAULT]); + } , error => { this.errorMessage = errorHandler(error); this.errorOpen = true; diff --git a/src/portal/src/i18n/lang/en-us-lang.json b/src/portal/src/i18n/lang/en-us-lang.json index 138953cc8..9b8fbc58d 100644 --- a/src/portal/src/i18n/lang/en-us-lang.json +++ b/src/portal/src/i18n/lang/en-us-lang.json @@ -729,12 +729,12 @@ "VERIFY_CERT": "Authentication Verify Cert" }, "OIDC": { - "OIDC_PROVIDER": "OIDC provider", + "OIDC_PROVIDER": "OIDC Provider", "ENDPOINT": "OIDC Endpoint", "CLIENT_ID": "OIDC Client ID", "CLIENTSECRET": "OIDC Client Secret", "SCOPE": "OIDC Scope", - "OIDCSKIPCERTVERIFY": "OIDC Verify Cert", + "OIDCSKIPCERTVERIFY": "OIDC Skip Verifying Certificate", "OIDC_SETNAME": "Set OIDC Username", "OIDC_SETNAMECONTENT": "You must create a Harbor username the first time when authenticating via a third party(OIDC).This will be used within Harbor to be associated with projects, roles, etc.", "OIDC_USERNAME": "Username" diff --git a/src/portal/src/i18n/lang/es-es-lang.json b/src/portal/src/i18n/lang/es-es-lang.json index 6b9b1c2e1..a8a09bb4d 100644 --- a/src/portal/src/i18n/lang/es-es-lang.json +++ b/src/portal/src/i18n/lang/es-es-lang.json @@ -734,7 +734,7 @@ "CLIENT_ID": "ID de cliente OIDC", "CLIENTSECRET": "OIDC Client Secret", "SCOPE": "OIDC Ámbito", - "OIDCSKIPCERTVERIFY": "OIDC Verify Cert", + "OIDCSKIPCERTVERIFY": "OIDC Skip Verificar certificado", "OIDC_SETNAME": "Set OIDC nombre de usuario", "OIDC_SETNAMECONTENT": "Usted debe crear un Harbor nombre de usuario la primera vez cuando la autenticación a través de un tercero (OIDC). Esta será usada en Harbor para ser asociados con proyectos, funciones, etc.", "OIDC_USERNAME": "Usuario" diff --git a/src/portal/src/i18n/lang/fr-fr-lang.json b/src/portal/src/i18n/lang/fr-fr-lang.json index 19da831da..8ea812ff7 100644 --- a/src/portal/src/i18n/lang/fr-fr-lang.json +++ b/src/portal/src/i18n/lang/fr-fr-lang.json @@ -694,12 +694,12 @@ "VERIFY_CERT": "authentification vérifier cert" }, "OIDC": { - "OIDC_PROVIDER": "OIDC fournisseur", - "ENDPOINT": "OIDC paramètre", + "OIDC_PROVIDER": "OIDC Fournisseur", + "ENDPOINT": "OIDC Faramètre", "CLIENT_ID": "no d'identification du client OIDC", "CLIENTSECRET": "OIDC Client Secret", "SCOPE": "OIDC Scope", - "OIDCSKIPCERTVERIFY": "OIDC vérifier cert", + "OIDCSKIPCERTVERIFY": "Certificat OIDC skip vérifier", "OIDC_SETNAME": "Ensemble OIDC nom d'utilisateur", "OIDC_SETNAMECONTENT": "vous devez créer un Harbor identifiant la première fois lors de la vérification par une tierce partie (oidc). il sera utilisé au sein de port à être associés aux projets, des rôles, etc.", "OIDC_USERNAME": "d'utilisateur" diff --git a/src/portal/src/i18n/lang/pt-br-lang.json b/src/portal/src/i18n/lang/pt-br-lang.json index 146ee58ac..9537e9762 100644 --- a/src/portal/src/i18n/lang/pt-br-lang.json +++ b/src/portal/src/i18n/lang/pt-br-lang.json @@ -728,7 +728,7 @@ "CLIENT_ID": "ID de cliente OIDC", "CLIENTSECRET": "OIDC Client Secret", "SCOPE": "Escopo OIDC", - "OIDCSKIPCERTVERIFY": "Verificar certificado OIDC", + "OIDCSKIPCERTVERIFY": "OIDC Skip Verificar Certificado", "OIDC_SETNAME": "Definir o Utilizador OIDC", "OIDC_SETNAMECONTENT": "Você deve Criar um Nome de usuário do Porto a primeira vez que autenticar através de um terceiro (OIDC). Isto será usado Dentro de Harbor para ser associado a projetos, papéis, etc.", "OIDC_USERNAME": "Utilizador" diff --git a/src/portal/src/i18n/lang/zh-cn-lang.json b/src/portal/src/i18n/lang/zh-cn-lang.json index 6d0104c30..58c7e0051 100644 --- a/src/portal/src/i18n/lang/zh-cn-lang.json +++ b/src/portal/src/i18n/lang/zh-cn-lang.json @@ -732,7 +732,7 @@ "ENDPOINT": "OIDC Endpoint", "CLIENT_ID": "OIDC 客户端标识", "CLIENTSECRET": "OIDC 客户端密码", - "SCOPE": "OIDC scope", + "SCOPE": "OIDC Scope", "OIDCSKIPCERTVERIFY": "OIDC 验证证书", "OIDC_SETNAME": "设置OIDC用户名", "OIDC_SETNAMECONTENT": "在通过第三方(OIDC)进行身份验证时,您必须第一次创建一个Harbor用户名。这将在端口中用于与项目、角色等关联。", diff --git a/tests/hostcfg.sh b/tests/hostcfg.sh index e1109fe13..9aae9c3a0 100755 --- a/tests/hostcfg.sh +++ b/tests/hostcfg.sh @@ -1,7 +1,9 @@ #!/bin/bash IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'` -PROTOCOL='https' #echo $IP sudo sed "s/reg.mydomain.com/$IP/" -i make/harbor.yml -sudo sed "s/^ui_url_protocol: .*/ui_url_protocol: $PROTOCOL/g" -i make/harbor.yml + +echo "https:" >> make/harbor.yml +echo " certificate: /data/cert/server.crt" >> make/harbor.yml +echo " private_key: /data/cert/server.key" >> make/harbor.yml \ No newline at end of file diff --git a/tests/resources/Harbor-Pages/Project-Members.robot b/tests/resources/Harbor-Pages/Project-Members.robot index f6e5fcebf..f8f5d3a3f 100644 --- a/tests/resources/Harbor-Pages/Project-Members.robot +++ b/tests/resources/Harbor-Pages/Project-Members.robot @@ -96,9 +96,7 @@ Add Guest Member To Project #select guest Mouse Down xpath=${project_member_guest_radio_checkbox} Mouse Up xpath=${project_member_guest_radio_checkbox} - Retry Button Click xpath=${project_member_add_confirmation_ok_xpath} - Retry Wait Element xpath=${project_member_xpath} - Sleep 1 + Retry Double Keywords When Error Retry Element Click xpath=${project_member_add_confirmation_ok_xpath} Retry Wait Until Page Not Contains Element xpath=${project_member_add_confirmation_ok_xpath} Delete Project Member [arguments] ${member} diff --git a/tests/resources/Harbor-Pages/Project.robot b/tests/resources/Harbor-Pages/Project.robot index 7829e15c6..7b4dde0da 100644 --- a/tests/resources/Harbor-Pages/Project.robot +++ b/tests/resources/Harbor-Pages/Project.robot @@ -23,19 +23,13 @@ ${HARBOR_VERSION} v1.1.1 Create An New Project [Arguments] ${projectname} ${public}=false Navigate To Projects - ${element_create_project_button}= Set Variable xpath=${create_project_button_xpath} - Wait Until Element Is Visible And Enabled ${element_create_project_button} - Click Button ${element_create_project_button} + Retry Button Click xpath=${create_project_button_xpath} Log To Console Project Name: ${projectname} - ${elemen_project_name}= Set Variable xpath=${project_name_xpath} - Wait Until Element Is Visible And Enabled ${elemen_project_name} - Input Text ${elemen_project_name} ${projectname} + Retry Text Input xpath=${project_name_xpath} ${projectname} ${element_project_public}= Set Variable xpath=${project_public_xpath} Run Keyword If '${public}' == 'true' Run Keywords Wait Until Element Is Visible And Enabled ${element_project_public} AND Click Element ${element_project_public} - ${element_create_project_OK_button_xpath}= Set Variable ${create_project_OK_button_xpath} - Wait Until Element Is Visible And Enabled ${element_create_project_OK_button_xpath} - Click Element ${element_create_project_OK_button_xpath} - Wait Until Page Does Not Contain Element ${create_project_CANCEL_button_xpath} + Retry Element Click ${create_project_OK_button_xpath} + Retry Wait Until Page Not Contains Element ${create_project_CANCEL_button_xpath} Go Into Project ${projectname} has_image=${false} Create An New Project With New User @@ -48,42 +42,37 @@ Create An New Project With New User #It's the log of project. Go To Project Log - Click Element xpath=${project_log_xpath} + Retry Element Click xpath=${project_log_xpath} Sleep 2 Switch To Member Sleep 3 - Click Element xpath=${project_member_xpath} + Retry Element Click xpath=${project_member_xpath} Sleep 1 Switch To Log - Wait Until Element Is Enabled xpath=${log_xpath} - Wait Until Element Is Visible xpath=${log_xpath} - Click Element xpath=${log_xpath} + Retry Element Click xpath=${log_xpath} Sleep 1 Switch To Replication - Click Element xpath=${project_replication_xpath} + Retry Element Click xpath=${project_replication_xpath} Sleep 1 Navigate To Projects - ${element}= Set Variable xpath=${projects_xpath} - Wait Until Element Is Visible And Enabled ${element} - Click Element ${element} + Retry Element Click xpath=${projects_xpath} Sleep 2 Project Should Display [Arguments] ${projectname} - ${element}= Set Variable xpath=//project//list-project//clr-dg-cell/a[contains(.,'${projectname}')] - Wait Until Element Is Visible And Enabled ${element} + Retry Wait Element xpath=//project//list-project//clr-dg-cell/a[contains(.,'${projectname}')] Project Should Not Display [Arguments] ${projectname} - Page Should Not Contain Element xpath=//project//list-project//clr-dg-cell/a[contains(.,'${projectname}')] + Retry Wait Until Page Not Contains Element xpath=//project//list-project//clr-dg-cell/a[contains(.,'${projectname}')] Search Private Projects - Click element xpath=//select - Click element xpath=//select/option[@value=1] + Retry Element Click xpath=//select + Retry Element Click xpath=//select/option[@value=1] Sleep 1 Capture Page Screenshot SearchPrivateProjects.png @@ -122,71 +111,53 @@ Delete Repo on CardView Delete Project [Arguments] ${projectname} Navigate To Projects - Sleep 1 - Click Element xpath=//clr-dg-row[contains(.,'${projectname}')]//clr-checkbox-wrapper//label - Sleep 1 - Click Element xpath=//button[contains(.,'Delete')] - Sleep 2 - Click Element //clr-modal//button[contains(.,'DELETE')] + Retry Element Click xpath=//clr-dg-row[contains(.,'${projectname}')]//clr-checkbox-wrapper//label + Retry Element Click xpath=//button[contains(.,'Delete')] + Retry Element Click //clr-modal//button[contains(.,'DELETE')] Sleep 1 Project Should Not Be Deleted [Arguments] ${projname} Delete Project ${projname} - Sleep 1 - Page Should Contain Element //clr-tab-content//div[contains(.,'${projname}')]/../div/clr-icon[@shape='error-standard'] + Retry Wait Until Page Contains Element //clr-tab-content//div[contains(.,'${projname}')]/../div/clr-icon[@shape='error-standard'] Project Should Be Deleted [Arguments] ${projname} Delete Project ${projname} - Sleep 2 - Page Should Contain Element //clr-tab-content//div[contains(.,'${projname}')]/../div/clr-icon[@shape='success-standard'] + Retry Wait Until Page Contains Element //clr-tab-content//div[contains(.,'${projname}')]/../div/clr-icon[@shape='success-standard'] Advanced Search Should Display - Page Should Contain Element xpath=//audit-log//div[@class='flex-xs-middle']/button + Retry Wait Until Page Contains Element xpath=//audit-log//div[@class='flex-xs-middle']/button # it's not a common keywords, only used into log case. Do Log Advanced Search Capture Page Screenshot LogAdvancedSearch.png - Sleep 1 - Page Should Contain Element xpath=//clr-dg-row[contains(.,'pull')] - Page Should Contain Element xpath=//clr-dg-row[contains(.,'push')] - Page Should Contain Element xpath=//clr-dg-row[contains(.,'create')] - Page Should Contain Element xpath=//clr-dg-row[contains(.,'delete')] - Sleep 1 - Click Element xpath=//audit-log//div[@class='flex-xs-middle']/button - Sleep 1 - Click Element xpath=//project-detail//audit-log//clr-dropdown/button - Sleep 1 + Retry Wait Until Page Contains Element xpath=//clr-dg-row[contains(.,'pull')] + Retry Wait Until Page Contains Element xpath=//clr-dg-row[contains(.,'push')] + Retry Wait Until Page Contains Element xpath=//clr-dg-row[contains(.,'create')] + Retry Wait Until Page Contains Element xpath=//clr-dg-row[contains(.,'delete')] + Retry Element Click xpath=//audit-log//div[@class='flex-xs-middle']/button + Retry Element Click xpath=//project-detail//audit-log//clr-dropdown/button #pull log - Sleep 1 - Click Element xpath=//audit-log//clr-dropdown//a[contains(.,'Pull')] - Sleep 1 - Page Should Not Contain Element xpath=//clr-dg-row[contains(.,'pull')] + Retry Element Click xpath=//audit-log//clr-dropdown//a[contains(.,'Pull')] + Retry Wait Until Page Not Contains Element xpath=//clr-dg-row[contains(.,'pull')] #push log - Click Element xpath=//audit-log//clr-dropdown/button - Sleep 1 - Click Element xpath=//audit-log//clr-dropdown//a[contains(.,'Push')] - Sleep 1 - Page Should Not Contain Element xpath=//clr-dg-row[contains(.,'push')] + Retry Element Click xpath=//audit-log//clr-dropdown/button + Retry Element Click xpath=//audit-log//clr-dropdown//a[contains(.,'Push')] + Retry Wait Until Page Not Contains Element xpath=//clr-dg-row[contains(.,'push')] #create log - Click Element xpath=//audit-log//clr-dropdown/button - Sleep 1 - Click Element xpath=//audit-log//clr-dropdown//a[contains(.,'Create')] - Sleep 1 - Page Should Not Contain Element xpath=//clr-dg-row[contains(.,'create')] + Retry Element Click xpath=//audit-log//clr-dropdown/button + Retry Element Click xpath=//audit-log//clr-dropdown//a[contains(.,'Create')] + Retry Wait Until Page Not Contains Element xpath=//clr-dg-row[contains(.,'create')] #delete log - Click Element xpath=//audit-log//clr-dropdown/button - Sleep 1 - Click Element xpath=//audit-log//clr-dropdown//a[contains(.,'Delete')] - Sleep 1 - Page Should Not Contain Element xpath=//clr-dg-row[contains(.,'delete')] + Retry Element Click xpath=//audit-log//clr-dropdown/button + Retry Element Click xpath=//audit-log//clr-dropdown//a[contains(.,'Delete')] + Retry Wait Until Page Not Contains Element xpath=//clr-dg-row[contains(.,'delete')] #others - Click Element xpath=//audit-log//clr-dropdown/button - Click Element xpath=//audit-log//clr-dropdown//a[contains(.,'Others')] - Sleep 1 - Click Element xpath=//audit-log//hbr-filter//clr-icon - Input Text xpath=//audit-log//hbr-filter//input harbor + Retry Element Click xpath=//audit-log//clr-dropdown/button + Retry Element Click xpath=//audit-log//clr-dropdown//a[contains(.,'Others')] + Retry Element Click xpath=//audit-log//hbr-filter//clr-icon + Retry Text Input xpath=//audit-log//hbr-filter//input harbor Sleep 1 Capture Page Screenshot LogAdvancedSearch2.png ${rc} = Get Matching Xpath Count //audit-log//clr-dg-row @@ -204,74 +175,61 @@ Go Into Repo Capture Page Screenshot gointo_${repoName}.png Switch To CardView - Sleep 2 - Click Element xpath=//hbr-repository-gridview//span[@class='card-btn']/clr-icon + Retry Element Click xpath=//hbr-repository-gridview//span[@class='card-btn']/clr-icon Sleep 5 Expand Repo [Arguments] ${projectname} - Click Element //repository//clr-dg-row[contains(.,'${projectname}')]//button/clr-icon + Retry Element Click //repository//clr-dg-row[contains(.,'${projectname}')]//button/clr-icon Sleep 1 Edit Repo Info - Click Element //*[@id='repo-info'] - Sleep 1 - Page Should Contain Element //*[@id='info']/form/div[2] + Retry Element Click //*[@id='repo-info'] + Retry Wait Until Page Contains Element //*[@id='info']/form/div[2] # Cancel input - Click Element xpath=//*[@id='info-edit-button']/button + Retry Element Click xpath=//*[@id='info-edit-button']/button Input Text xpath=//*[@id='info']/form/div[2]/textarea test_description_info - Click Element xpath=//*[@id='info']/form/div[3]/button[2] - Sleep 1 - Click Element xpath=//*[@id='info']/form/confirmation-dialog/clr-modal/div/div[1]/div[1]/div/div[3]/button[2] - Sleep 1 - Page Should Contain Element //*[@id='info']/form/div[2] + Retry Element Click xpath=//*[@id='info']/form/div[3]/button[2] + Retry Element Click xpath=//*[@id='info']/form/confirmation-dialog/clr-modal/div/div[1]/div[1]/div/div[3]/button[2] + Retry Wait Until Page Contains Element //*[@id='info']/form/div[2] # Confirm input - Click Element xpath=//*[@id='info-edit-button']/button + Retry Element Click xpath=//*[@id='info-edit-button']/button Input Text xpath=//*[@id='info']/form/div[2]/textarea test_description_info - Click Element xpath=//*[@id='info']/form/div[3]/button[1] - Sleep 1 - Page Should Contain test_description_info + Retry Element Click xpath=//*[@id='info']/form/div[3]/button[1] + Retry Wait Until Page Contains test_description_info Capture Page Screenshot RepoInfo.png Switch To Project Label - Click Element xpath=//project-detail//a[contains(.,'Labels')] + Retry Element Click xpath=//project-detail//a[contains(.,'Labels')] Sleep 1 Switch To Project Repo - Click Element xpath=//project-detail//a[contains(.,'Repositories')] + Retry Element Click xpath=//project-detail//a[contains(.,'Repositories')] Sleep 1 Add Labels To Tag [Arguments] ${tagName} ${labelName} - Click Element xpath=//clr-dg-row[contains(.,'${tagName}')]//label + Retry Element Click xpath=//clr-dg-row[contains(.,'${tagName}')]//label Capture Page Screenshot add_${labelName}.png - Sleep 1 - Click Element xpath=//clr-dg-action-bar//clr-dropdown//button - Sleep 1 - Click Element xpath=//clr-dropdown//div//label[contains(.,'${labelName}')] - Sleep 3 - Page Should Contain Element xpath=//clr-dg-row//label[contains(.,'${labelName}')] + Retry Element Click xpath=//clr-dg-action-bar//clr-dropdown//button + Retry Element Click xpath=//clr-dropdown//div//label[contains(.,'${labelName}')] + Retry Wait Until Page Contains Element xpath=//clr-dg-row//label[contains(.,'${labelName}')] Filter Labels In Tags [Arguments] ${labelName1} ${labelName2} - Sleep 2 - Click Element xpath=//*[@id='filterArea']//hbr-filter/span/clr-icon - Sleep 2 - Page Should Contain Element xpath=//*[@id='filterArea']//div//button[contains(.,'${labelName1}')] - Click Element xpath=//*[@id='filterArea']//div//button[contains(.,'${labelName1}')] - Sleep 2 - Click Element xpath=//*[@id='filterArea']//hbr-filter/span/clr-icon - Page Should Contain Element xpath=//clr-datagrid//label[contains(.,'${labelName1}')] + Retry Element Click xpath=//*[@id='filterArea']//hbr-filter/span/clr-icon + Retry Wait Until Page Contains Element xpath=//*[@id='filterArea']//div//button[contains(.,'${labelName1}')] + Retry Element Click xpath=//*[@id='filterArea']//div//button[contains(.,'${labelName1}')] + Retry Element Click xpath=//*[@id='filterArea']//hbr-filter/span/clr-icon + Retry Wait Until Page Contains Element xpath=//clr-datagrid//label[contains(.,'${labelName1}')] - Click Element xpath=//*[@id='filterArea']//hbr-filter/span/clr-icon - Sleep 2 - Click Element xpath=//*[@id='filterArea']//div//button[contains(.,'${labelName2}')] - Sleep 2 - Click Element xpath=//*[@id='filterArea']//hbr-filter/span/clr-icon + Retry Element Click xpath=//*[@id='filterArea']//hbr-filter/span/clr-icon + Retry Element Click xpath=//*[@id='filterArea']//div//button[contains(.,'${labelName2}')] + Retry Element Click xpath=//*[@id='filterArea']//hbr-filter/span/clr-icon Sleep 2 Capture Page Screenshot filter_${labelName2}.png - Page Should Contain Element xpath=//clr-dg-row[contains(.,'${labelName2}')] - Page Should Not Contain Element xpath=//clr-dg-row[contains(.,'${labelName1}')] + Retry Wait Until Page Contains Element xpath=//clr-dg-row[contains(.,'${labelName2}')] + Retry Wait Until Page Not Contains Element xpath=//clr-dg-row[contains(.,'${labelName1}')] Get Statics Private Repo ${privaterepo}= Get Text //project/div/div/div[1]/div/statistics-panel/div/div[2]/div[1]/div[2]/div[2]/statistics/div/span[1] diff --git a/tests/resources/Util.robot b/tests/resources/Util.robot index a30977735..d844c4a94 100644 --- a/tests/resources/Util.robot +++ b/tests/resources/Util.robot @@ -176,7 +176,7 @@ Retry Keyword When Error Retry Double Keywords When Error [Arguments] ${keyword1} ${element1} ${keyword2} ${element2} :For ${n} IN RANGE 1 6 - \ Log To Console Trying Delete Repo ${n} times ... + \ Log To Console Trying ${keyword1} and ${keyword2} ${n} times ... \ ${out1} Run Keyword And Ignore Error ${keyword1} ${element1} \ Capture Page Screenshot \ ${out2} Run Keyword And Ignore Error ${keyword2} ${element2} diff --git a/tests/testprepare.sh b/tests/testprepare.sh index 264270ad0..3776b597a 100755 --- a/tests/testprepare.sh +++ b/tests/testprepare.sh @@ -8,7 +8,7 @@ cp /data/secret/core/private_key.pem /etc/core/ mkdir src/core/conf cp make/common/config/core/app.conf src/core/conf/ if [ "$(uname)" == "Darwin" ]; then - IP=`ifconfig en0 | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}'` + IP=`ifconfig en0 | grep "inet " | grep -Fv 127.0.0.1 | awk '{print $2}'` else IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'` fi diff --git a/tests/travis/api_common_install.sh b/tests/travis/api_common_install.sh index 3d06c8081..504157733 100644 --- a/tests/travis/api_common_install.sh +++ b/tests/travis/api_common_install.sh @@ -10,12 +10,12 @@ sudo sed "s/127.0.0.1/$1/" -i tests/generateCerts.sh sudo ./tests/generateCerts.sh sudo mkdir -p /etc/docker/certs.d/$1 && sudo cp ./harbor_ca.crt /etc/docker/certs.d/$1/ +sudo ./tests/hostcfg.sh + if [ "$2" = 'LDAP' ]; then - sudo ./tests/hostcfg.sh LDAP cd tests && sudo ./ldapprepare.sh && cd .. fi -sudo ./tests/hostcfg.sh # prepare a chart file for API_DB test...