Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update migratrion script (#7728) 

* Fix migration script

1. port is string when parsed from configparser
2. remove index and db_user in if condition

Signed-off-by: Qian Deng <dengq@vmware.com>

* Add port to public_url

Add port to public_url

Signed-off-by: Qian Deng <dengq@vmware.com>

* Customized value for notary and clair

db config in notary and clair is hardcoded

Signed-off-by: Qian Deng <dengq@vmware.com>

* Add notary and clair db config in harbor.yml

Add notary clair config to harbor.yml and fix related regression

Signed-off-by: Qian Deng <dengq@vmware.com>
This commit is contained in:
Qian Deng 2019-05-10 10:44:05 +08:00 committed by Wang Yan
parent 4218206b59
commit bb66358df8
14 changed files with 143 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
make/harbor.yml
View File

@ -83,11 +83,34 @@ _version: 1.8.0
# Uncomment external_database if using external database. And the password will replace the the password setting in database.
# And currently only support postgres.
# external_database:
# host: postgresql
# port: 5432
# username: postgres
# password: root123
# ssl_mode: disable
# harbor:
# host: postgresql
# port: 5432
# db_name: registry
# username: postgres
# password: root123
# ssl_mode: disable
# clair:
# host: postgresql
# port: 5432
# db_name: registry
# username: postgres
# password: root123
# ssl_mode: disable
# notary_signer:
# host: postgresql
# port: 5432
# db_name: registry
# username: postgres
# password: root123
# ssl_mode: disable
# notary_server:
# host: postgresql
# port: 5432
# db_name: registry
# username: postgres
# password: root123
# ssl_mode: disable
# Uncomment external_redis if using external Redis server
# external_redis:
@ -101,4 +124,4 @@ _version: 1.8.0
# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
# uaa:
# ca_file: /path/to/ca
# ca_file: /path/to/ca

4
make/photon/prepare/templates/clair/config.yaml.jinja
View File

@ -2,7 +2,7 @@ clair:
database:
type: pgsql
options:
source: postgresql://{{username}}:{{password}}@{{host}}:{{port}}/{{dbname}}?sslmode=disable
source: postgresql://{{clair_db_username}}:{{clair_db_password}}@{{clair_db_host}}:{{clair_db_port}}/{{clair_db_name}}?sslmode={{clair_db_sslmode}}
# Number of elements kept in the cache
# Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database.
@ -16,7 +16,7 @@ clair:
# Deadline before an API request will respond with a 503
timeout: 300s
updater:
interval: {{interval}}h
interval: {{clair_updaters_interval}}h
notifier:
attempts: 3

2
make/photon/prepare/templates/clair/postgres_env.jinja
View File

@ -1 +1 @@
POSTGRES_PASSWORD={{password}}
POSTGRES_PASSWORD={{clair_db_password}}

24
make/photon/prepare/templates/core/env.jinja
View File

@ -9,12 +9,12 @@ PORT=8080
LOG_LEVEL={{log_level}}
EXT_ENDPOINT={{public_url}}
DATABASE_TYPE=postgresql
POSTGRESQL_HOST={{db_host}}
POSTGRESQL_PORT={{db_port}}
POSTGRESQL_USERNAME={{db_user}}
POSTGRESQL_PASSWORD={{db_password}}
POSTGRESQL_DATABASE=registry
POSTGRESQL_SSLMODE=disable
POSTGRESQL_HOST={{harbor_db_host}}
POSTGRESQL_PORT={{harbor_db_port}}
POSTGRESQL_USERNAME={{harbor_db_username}}
POSTGRESQL_PASSWORD={{harbor_db_password}}
POSTGRESQL_DATABASE={{harbor_db_name}}
POSTGRESQL_SSLMODE={{harbor_db_sslmode}}
REGISTRY_URL={{registry_url}}
TOKEN_SERVICE_URL={{token_service_url}}
HARBOR_ADMIN_PASSWORD={{harbor_admin_password}}
@ -24,12 +24,12 @@ JOBSERVICE_SECRET={{jobservice_secret}}
ADMIRAL_URL={{admiral_url}}
WITH_NOTARY={{with_notary}}
WITH_CLAIR={{with_clair}}
CLAIR_DB_PASSWORD={{db_password}}
CLAIR_DB_HOST={{db_host}}
CLAIR_DB_PORT={{db_port}}
CLAIR_DB_USERNAME={{db_user}}
CLAIR_DB={{clair_db}}
CLAIR_DB_SSLMODE=disable
CLAIR_DB_PASSWORD={{clair_db_password}}
CLAIR_DB_HOST={{clair_db_host}}
CLAIR_DB_PORT={{clair_db_port}}
CLAIR_DB_USERNAME={{clair_db_username}}
CLAIR_DB={{clair_db_name}}
CLAIR_DB_SSLMODE={{clair_db_sslmode}}
CORE_URL={{core_url}}
JOBSERVICE_URL={{jobservice_url}}
CLAIR_URL={{clair_url}}

2
make/photon/prepare/templates/db/env.jinja
View File

@ -1 +1 @@
POSTGRES_PASSWORD={{db_password}}
POSTGRES_PASSWORD={{harbor_db_password}}

2
make/photon/prepare/templates/notary/server-config.postgres.json.jinja
View File

@ -14,7 +14,7 @@
},
"storage": {
"backend": "postgres",
"db_url": "postgres://server:password@postgresql:5432/notaryserver?sslmode=disable"
"db_url": "postgres://{{notary_server_db_username}}:{{notary_server_db_password}}@{{notary_server_db_host}}:{{notary_server_db_port}}/{{notary_server_db_name}}?sslmode={{notary_server_db_sslmode}}"
},
"auth": {
"type": "token",

2
make/photon/prepare/templates/notary/server_env.jinja
View File

@ -1,2 +1,2 @@
MIGRATIONS_PATH=migrations/server/postgresql
DB_URL=postgres://server:password@postgresql:5432/notaryserver?sslmode=disable
DB_URL=postgres://{{notary_server_db_username}}:{{notary_server_db_password}}@{{notary_server_db_host}}:{{notary_server_db_port}}/{{notary_server_db_name}}?sslmode={{notary_server_db_sslmode}}

4
make/photon/prepare/templates/notary/signer-config.postgres.json.jinja
View File

@ -9,7 +9,7 @@
},
"storage": {
"backend": "postgres",
"db_url": "postgres://signer:password@postgresql:5432/notarysigner?sslmode=disable",
"default_alias":"defaultalias"
"db_url": "postgres://{{notary_signer_db_username}}:{{notary_signer_db_password}}@{{notary_signer_db_host}}:{{notary_signer_db_port}}/{{notary_signer_db_name}}?sslmode={{notary_signer_db_sslmode}}",
"default_alias": "{{alias}}"
}
}

2
make/photon/prepare/templates/notary/signer_env.jinja
View File

@ -1,3 +1,3 @@
NOTARY_SIGNER_DEFAULTALIAS={{alias}}
MIGRATIONS_PATH=migrations/signer/postgresql
DB_URL=postgres://signer:password@postgresql:5432/notarysigner?sslmode=disable
DB_URL=postgres://{{notary_signer_db_username}}:{{notary_signer_db_password}}@{{notary_signer_db_host}}:{{notary_signer_db_port}}/{{notary_signer_db_name}}?sslmode={{notary_signer_db_sslmode}}

9
make/photon/prepare/utils/clair.py
View File

@ -27,19 +27,14 @@ def prepare_clair(config_dict):
render_jinja(
postgres_env_template,
postgres_env_path,
password=config_dict['db_password'])
**config_dict)
render_jinja(
clair_config_template,
clair_config_path,
uid=DEFAULT_UID,
gid=DEFAULT_GID,
password= config_dict['db_password'],
username= config_dict['db_user'],
host= config_dict['db_host'],
port= config_dict['db_port'],
dbname= config_dict['clair_db'],
interval= config_dict['clair_updaters_interval'])
**config_dict)
# config http proxy for Clair
render_jinja(

75
make/photon/prepare/utils/configs.py
View File

@ -79,16 +79,45 @@ def parse_yaml_config(config_file_path):
config_dict['cert_path'] = https_config["certificate"]
config_dict['cert_key_path'] = https_config["private_key"]
config_dict['public_url'] = configs.get('external_url') or '{protocol}://{hostname}'.format(**config_dict)
if configs.get('external_url'):
config_dict['public_url'] = configs.get('external_url')
else:
if config_dict['protocol'] == 'https':
config_dict['public_url'] = '{protocol}://{hostname}:{https_port}'.format(**config_dict)
else:
config_dict['public_url'] = '{protocol}://{hostname}:{http_port}'.format(**config_dict)
# DB configs
db_configs = configs.get('database')
if db_configs:
config_dict['db_host'] = 'postgresql'
config_dict['db_port'] = 5432
config_dict['db_user'] = 'postgres'
config_dict['db_password'] = db_configs.get("password") or ''
config_dict['ssl_mode'] = 'disable'
# harbor db
config_dict['harbor_db_host'] = 'postgresql'
config_dict['harbor_db_port'] = 5432
config_dict['harbor_db_name'] = 'registry'
config_dict['harbor_db_username'] = 'postgres'
config_dict['harbor_db_password'] = db_configs.get("password") or ''
config_dict['harbor_db_sslmode'] = 'disable'
# clari db
config_dict['clair_db_host'] = 'postgresql'
config_dict['clair_db_port'] = 5432
config_dict['clair_db_name'] = 'postgres'
config_dict['clair_db_username'] = 'postgres'
config_dict['clair_db_password'] = db_configs.get("password") or ''
config_dict['clair_db_sslmode'] = 'disable'
# notary signer
config_dict['notary_signer_db_host'] = 'postgresql'
config_dict['notary_signer_db_port'] = 5432
config_dict['notary_signer_db_name'] = 'notarysigner'
config_dict['notary_signer_db_username'] = 'signer'
config_dict['notary_signer_db_password'] = 'password'
config_dict['notary_signer_db_sslmode'] = 'disable'
# notary server
config_dict['notary_server_db_host'] = 'postgresql'
config_dict['notary_server_db_port'] = 5432
config_dict['notary_server_db_name'] = 'notaryserver'
config_dict['notary_server_db_username'] = 'server'
config_dict['notary_server_db_password'] = 'password'
config_dict['notary_server_db_sslmode'] = 'disable'
# Data path volume
@ -152,12 +181,34 @@ def parse_yaml_config(config_file_path):
# external DB, if external_db enabled, it will cover the database config
external_db_configs = configs.get('external_database') or {}
if external_db_configs:
config_dict['db_password'] = external_db_configs.get('password') or ''
config_dict['db_host'] = external_db_configs['host']
config_dict['db_port'] = external_db_configs['port']
config_dict['db_user'] = external_db_configs['username']
if external_db_configs.get('ssl_mode'):
config_dict['db_ssl_mode'] = external_db_configs['ssl_mode']
# harbor db
config_dict['harbor_db_host'] = external_db_configs['harbor']['host']
config_dict['harbor_db_port'] = external_db_configs['harbor']['port']
config_dict['harbor_db_name'] = external_db_configs['harbor']['db_name']
config_dict['harbor_db_username'] = external_db_configs['harbor']['username']
config_dict['harbor_db_password'] = external_db_configs['harbor']['password']
config_dict['harbor_db_sslmode'] = external_db_configs['harbor']['ssl_mode']
# clari db
config_dict['clair_db_host'] = external_db_configs['clair']['host']
config_dict['clair_db_port'] = external_db_configs['clair']['port']
config_dict['clair_db_name'] = external_db_configs['clair']['db_name']
config_dict['clair_db_username'] = external_db_configs['clair']['username']
config_dict['clair_db_password'] = external_db_configs['clair']['password']
config_dict['clair_db_sslmode'] = external_db_configs['clair']['ssl_mode']
# notary signer
config_dict['notary_signer_db_host'] = external_db_configs['notary_signer']['host']
config_dict['notary_signer_db_port'] = external_db_configs['notary_signer']['port']
config_dict['notary_signer_db_name'] = external_db_configs['notary_signer']['db_name']
config_dict['notary_signer_db_username'] = external_db_configs['notary_signer']['username']
config_dict['notary_signer_db_password'] = external_db_configs['notary_signer']['password']
config_dict['notary_signer_db_sslmode'] = external_db_configs['notary_signer']['ssl_mode']
# notary server
config_dict['notary_server_db_host'] = external_db_configs['notary_server']['host']
config_dict['notary_server_db_port'] = external_db_configs['notary_server']['port']
config_dict['notary_server_db_name'] = external_db_configs['notary_server']['db_name']
config_dict['notary_server_db_username'] = external_db_configs['notary_server']['username']
config_dict['notary_server_db_password'] = external_db_configs['notary_server']['password']
config_dict['notary_server_db_sslmode'] = external_db_configs['notary_server']['ssl_mode']
# redis config

2
make/photon/prepare/utils/db.py
View File

@ -14,7 +14,7 @@ def prepare_db(config_dict):
render_jinja(
db_env_template_path,
db_conf_env,
db_password=config_dict['db_password'])
harbor_db_password=config_dict['harbor_db_password'])
def prepare_db_config_dir():
prepare_config_dir(db_config_dir)

49
make/photon/prepare/utils/notary.py
View File

@ -70,10 +70,6 @@ def prepare_env_notary(nginx_config_dir):
else:
raise(Exception("No certs for notary"))
# copy server_env to notary config
shutil.copy2(
os.path.join(notary_template_dir, "server_env.jinja"),
os.path.join(notary_config_dir, "server_env"))
print("Copying nginx configuration file for notary")
shutil.copy2(
@ -90,32 +86,37 @@ def prepare_notary(config_dict, nginx_config_dir, ssl_cert_path, ssl_cert_key_pa
prepare_env_notary(nginx_config_dir)
render_jinja(
notary_signer_pg_template,
notary_signer_pg_config,
uid=DEFAULT_UID,
gid=DEFAULT_GID
)
notary_server_nginx_config_template,
os.path.join(nginx_config_dir, "notary.server.conf"),
ssl_cert=ssl_cert_path,
ssl_cert_key=ssl_cert_key_path)
render_jinja(
notary_server_pg_template,
notary_server_pg_config,
uid=DEFAULT_UID,
gid=DEFAULT_GID,
token_endpoint=config_dict['public_url'])
render_jinja(
notary_server_nginx_config_template,
os.path.join(nginx_config_dir, "notary.server.conf"),
ssl_cert=ssl_cert_path,
ssl_cert_key=ssl_cert_key_path)
default_alias = get_alias(secret_key_dir)
render_jinja(
notary_signer_env_template,
notary_signer_env_path,
alias=default_alias)
token_endpoint=config_dict['public_url'],
**config_dict)
render_jinja(
notary_server_env_template,
notary_server_env_path
)
notary_server_env_path,
**config_dict
)
default_alias = get_alias(secret_key_dir)
render_jinja(
notary_signer_env_template,
notary_signer_env_path,
alias=default_alias,
**config_dict)
render_jinja(
notary_signer_pg_template,
notary_signer_pg_config,
uid=DEFAULT_UID,
gid=DEFAULT_GID,
alias=default_alias,
**config_dict)

4
tools/migration/cfg/migrator_1_8_0/__init__.py
View File

@ -34,7 +34,7 @@ def migrate(input_cfg, output_cfg):
val = {}
for k in keys:
val[k] = d.get(k,'')
if val['db_host'] == 'postgresql' and val['db_port'] == 5432 and val['db_user'] == 'postgres':
if val['db_host'] == 'postgresql' and val['db_port'] == '5432':
val['external_db'] = False
else:
val['external_db'] = True
@ -46,7 +46,7 @@ def migrate(input_cfg, output_cfg):
val['registry_storage_provider_name'],
val['registry_storage_provider_config']
)
if val['redis_host'] == 'redis' and val['redis_port'] == 6379 and not val['redis_password'] and val['redis_db_index'] == '1,2,3':
if val['redis_host'] == 'redis' and val['redis_port'] == '6379':
val['external_redis'] = False
else:
val['registry_db_index'], val['jobservice_db_index'], val['chartmuseum_db_index'] = map(int, val['redis_db_index'].split(','))