Feat: Upgrade configs to harbor 2.0

add migrate files for harbor 2.0

Signed-off-by: DQ <dengq@vmware.com>

make/harbor.yml.tmpl

@@ -125,7 +125,7 @@ log:
   #   port: 5140
 
 #This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
-_version: 1.10.0
+_version: 2.0.0
 
 # Uncomment external_database if using external database.
 # external_database:

make/photon/prepare/commands/migrate.py

@@ -9,13 +9,12 @@ from migration.utils import read_conf, to_module_path, search
 @click.command()
 @click.option('-i', '--input', 'input_', default='', help="The path of original config file")
 @click.option('-o', '--output', default='', help="the path of output config file")
-@click.option('-t', '--target', default='1.10.0', help="target version of input path")
+@click.option('-t', '--target', default='2.0.0', help="target version of input path")
 def migrate(input_, output, target):
+    if not output:
+        output = input_
     input_path = get_realpath(input_)
-    if output == '':
-        output_path = input_path
-    else:
-        output_path = get_realpath(output)
+    output_path = get_realpath(output)
 
     configs = read_conf(input_path)
 
@@ -29,11 +28,10 @@ def migrate(input_, output, target):
     for m in search(input_version, target):
         current_output_path = "harbor.yml.{}.tmp".format(m.revision)
         click.echo("migrating to version {}".format(m.revision))
-        print("migrating to version {}".format(m.revision))
         m.migrate(current_input_path, current_output_path)
         current_input_path = current_output_path
     shutil.copy(current_input_path, output_path)
-    print("Written new values to %s" % output_path)
+    click.echo("Written new values to {}".format(output))
     for tmp_f in glob.glob("harbor.yml.*.tmp"):
         os.remove(tmp_f)
 

make/photon/prepare/migration/utils.py

@@ -19,12 +19,15 @@ def read_conf(path):
 def to_module_path(ver):
     return "migration.versions.{}".format(ver.replace(".","_"))
 
-def search(input_ver: str, target_ver: str):
+def search(input_ver: str, target_ver: str) -> list:
+    """
+    Search accept a input version and the target version.
+    Returns the module of migrations in the upgrade path
+    """
     def helper():
         nonlocal basedir
         nonlocal cur_target
         while True:
-            print(input_ver, cur_target)
             module_path = to_module_path(cur_target)
             if os.path.isdir(os.path.join(basedir, 'versions', cur_target.replace(".","_"))):
                 module = importlib.import_module(module_path)
@@ -36,7 +39,7 @@ def search(input_ver: str, target_ver: str):
                 else:
                     return
             else:
-                print(os.path.join(basedir, 'versions', cur_target.replace(".","_")), 'not dir')
+                click.echo('{} not dir'.format(os.path.join(basedir, 'versions', cur_target.replace(".","_"))))
                 return
     basedir = os.path.dirname(__file__)
     cur_target = target_ver

make/photon/prepare/migration/versions/2_0_0/__init__.py

@@ -0,0 +1,23 @@
+import os
+
+import yaml
+from jinja2 import Environment, FileSystemLoader, StrictUndefined
+
+from migration import utils
+
+revision = '2.0.0'
+down_revision = '1.10.0'
+
+def migrate(input_cfg, output_cfg):
+    config_dict = utils.read_conf(input_cfg)
+
+    current_dir = os.path.dirname(__file__)
+    tpl = Environment(
+        loader=FileSystemLoader(current_dir),
+        undefined=StrictUndefined,
+        trim_blocks=True,
+        lstrip_blocks=True
+        ).get_template('harbor.yml.jinja')
+
+    with open(output_cfg, 'w') as f:
+        f.write(tpl.render(**config_dict))

make/photon/prepare/migration/versions/2_0_0/harbor.yml.jinja

@@ -0,0 +1,387 @@
+# Configuration file of Harbor
+
+# The IP address or hostname to access admin UI and registry service.
+# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
+hostname: {{ hostname }}
+
+# http related config
+{% if http is defined %}
+http:
+  # port for http, default is 80. If https enabled, this port will redirect to https port
+  port: {{ http.port }}
+{% else %}
+# http:
+#   # port for http, default is 80. If https enabled, this port will redirect to https port
+#   port: 80
+{% endif %}
+
+{% if https is defined %}
+# https related config
+https:
+  # https port for harbor, default is 443
+  port: {{ https.port }}
+  # The path of cert and key files for nginx
+  certificate: {{ https.certificate }}
+  private_key: {{ https.private_key }}
+{% else %}
+# https related config
+# https:
+#   # https port for harbor, default is 443
+#   port: 443
+#   # The path of cert and key files for nginx
+#   certificate: /your/certificate/path
+#   private_key: /your/private/key/path
+{% endif %}
+
+{% if internal_tls is defined %}
+internal_tls:
+  # set enabled to true means internal tls is enabled
+  enabled: {{ internal_tls.enabled }}
+  # verify_client_cert used to decide whether verify client certificate
+  verify_client_cert: {{ internal_tls.verify_client_cert }}
+  # put your cert and key files on dir
+  dir: {{ internal_tls.dir }}
+{% else %}
+# # Uncomment following will enable tls communication between all harbor components
+# internal_tls:
+#   # set enabled to true means internal tls is enabled
+#   enabled: true
+#   # verify_client_cert used to decide whether verify client certificate
+#   verify_client_cert: false
+#   # put your cert and key files on dir
+#   dir: /etc/harbor/tls/internal
+{% endif %}
+
+# Uncomment external_url if you want to enable external proxy
+# And when it enabled the hostname will no longer used
+{% if external_url is defined %}
+external_url: {{ external_url }}
+{% else %}
+# external_url: https://reg.mydomain.com:8433
+{% endif %}
+
+# The initial password of Harbor admin
+# It only works in first time to install harbor
+# Remember Change the admin password from UI after launching Harbor.
+{% if harbor_admin_password is defined %}
+harbor_admin_password: {{ harbor_admin_password }}
+{% else %}
+harbor_admin_password: Harbor12345
+{% endif %}
+
+# Harbor DB configuration
+database:
+{% if database is defined %}
+  # The password for the root user of Harbor DB. Change this before any production use.
+  password: {{ database.password}}
+  # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
+  max_idle_conns: {{ database.max_idle_conns or 50}}
+  # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
+  # Note: the default number of connections is 100 for postgres.
+  max_open_conns: {{ database.max_open_conns or 100}}
+{% else %}
+  # The password for the root user of Harbor DB. Change this before any production use.
+  password: root123
+  # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
+  max_idle_conns: 50
+  # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
+  # Note: the default number of connections is 100 for postgres.
+  max_open_conns: 100
+{% endif %}
+
+{% if data_volume is defined %}
+# The default data volume
+data_volume: {{ data_volume }}
+{% else %}
+# The default data volume
+data_volume: /data
+{% endif %}
+
+# Harbor Storage settings by default is using /data dir on local filesystem
+# Uncomment storage_service setting If you want to using external storage
+{% if storage_service is defined %}
+storage_service:
+  {% for key, value in storage_service.items() %}
+    {% if key == 'ca_bundle' %}
+#   # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
+#   # of registry's and chart repository's containers.  This is usually needed when the user hosts a internal storage with self signed certificate.
+  ca_bundle: {{ value if value is not none else '' }}
+    {% elif key == 'redirect' %}
+#   # set disable to true when you want to disable registry redirect
+  redirect:
+    disabled: {{ value.disabled }}
+    {% else %}
+#   # storage backend, default is filesystem, options include filesystem, azure, gcs, s3, swift and oss
+#   # for more info about this configuration please refer https://docs.docker.com/registry/configuration/
+  {{ key }}:
+      {% for k, v in value.items() %}
+    {{ k }}: {{ v if v is not none else '' }}
+      {% endfor %}
+    {% endif %}
+  {% endfor %}
+{% else %}
+# Harbor Storage settings by default is using /data dir on local filesystem
+# Uncomment storage_service setting If you want to using external storage
+# storage_service:
+#   # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
+#   # of registry's and chart repository's containers.  This is usually needed when the user hosts a internal storage with self signed certificate.
+#   ca_bundle:
+
+#   # storage backend, default is filesystem, options include filesystem, azure, gcs, s3, swift and oss
+#   # for more info about this configuration please refer https://docs.docker.com/registry/configuration/
+#   filesystem:
+#     maxthreads: 100
+#   # set disable to true when you want to disable registry redirect
+#   redirect:
+#     disabled: false
+{% endif %}
+
+# Clair configuration
+clair:
+  # The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
+{% if clair is defined %}
+  updaters_interval: {{ clair.updaters_interval }}
+{% else %}
+  updaters_interval: 12
+{% endif %}
+
+# Trivy configuration
+trivy:
+  # github_token The GitHub access token to download Trivy DB
+  #
+  # Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases.
+  # It is downloaded by Trivy from the GitHub release page https://github.com/aquasecurity/trivy-db/releases and cached
+  # in the local file system (/home/scanner/.cache/trivy/db/trivy.db). In addition, the database contains the update
+  # timestamp so Trivy can detect whether it should download a newer version from the Internet or use the cached one.
+  # Currently, the database is updated every 12 hours and published as a new release to GitHub.
+  #
+  # Anonymous downloads from GitHub are subject to the limit of 60 requests per hour. Normally such rate limit is enough
+  # for production operations. If, for any reason, it's not enough, you could increase the rate limit to 5000
+  # requests per hour by specifying the GitHub access token. For more details on GitHub rate limiting please consult
+  # https://developer.github.com/v3/#rate-limiting
+  #
+  # You can create a GitHub token by following the instuctions in
+  # https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line
+  #
+  # github_token: xxx
+
+jobservice:
+  # Maximum number of job workers in job service
+{% if jobservice is defined %}
+  max_job_workers: {{ jobservice.max_job_workers }}
+{% else %}
+  max_job_workers: 10
+{% endif %}
+
+notification:
+  # Maximum retry count for webhook job
+{% if notification is defined %}
+  webhook_job_max_retry: {{ notification.webhook_job_max_retry}}
+{% else %}
+  webhook_job_max_retry: 10
+{% endif %}
+
+{% if chart is defined %}
+chart:
+  # Change the value of absolute_url to enabled can enable absolute url in chart
+  absolute_url: {{ chart.absolute_url if chart.absolute_url == 'enabled' else 'disabled' }}
+{% else %}
+chart:
+  # Change the value of absolute_url to enabled can enable absolute url in chart
+  absolute_url: disabled
+{% endif %}
+
+# Log configurations
+log:
+  # options are debug, info, warning, error, fatal
+{% if log is defined %}
+  level: {{ log.level }}
+  # configs for logs in local storage
+  local:
+    # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
+    rotate_count: {{ log.local.rotate_count }}
+    # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes.
+    # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G
+    # are all valid.
+    rotate_size: {{ log.local.rotate_size }}
+    # The directory on your host that store log
+    location: {{ log.local.location }}
+  {% if log.external_endpoint is defined%}
+  external_endpoint:
+    # protocol used to transmit log to external endpoint, options is tcp or udp
+    protocol: {{ log.external_endpoint.protocol }}
+    # The host of external endpoint
+    host: {{ log.external_endpoint.host }}
+    # Port of external endpoint
+    port: {{ log.external_endpoint.port }}
+  {% else %}
+    # Uncomment following lines to enable external syslog endpoint.
+    # external_endpoint:
+    #   # protocol used to transmit log to external endpoint, options is tcp or udp
+    #   protocol: tcp
+    #   # The host of external endpoint
+    #   host: localhost
+    #   # Port of external endpoint
+    #   port: 5140
+  {% endif %}
+{% else %}
+  level: info
+  # configs for logs in local storage
+  local:
+    # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
+    rotate_count: 50
+    # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes.
+    # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G
+    # are all valid.
+    rotate_size: 200M
+    # The directory on your host that store log
+    location: /var/log/harbor
+
+  # Uncomment following lines to enable external syslog endpoint.
+  # external_endpoint:
+  #   # protocol used to transmit log to external endpoint, options is tcp or udp
+  #   protocol: tcp
+  #   # The host of external endpoint
+  #   host: localhost
+  #   # Port of external endpoint
+  #   port: 5140
+{% endif %}
+
+
+#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
+_version: 2.0.0
+{% if external_database is defined %}
+# Uncomment external_database if using external database.
+external_database:
+  harbor:
+    host: {{ external_database.harbor.host }}
+    port: {{ external_database.harbor.port }}
+    db_name: {{ external_database.harbor.db_name }}
+    username: {{ external_database.harbor.username }}
+    password: {{ external_database.harbor.password }}
+    ssl_mode: {{ external_database.harbor.ssl_mode }}
+    max_idle_conns: {{ external_database.harbor.max_idle_conns }}
+    max_open_conns: {{ external_database.harbor.max_open_conns }}
+  clair:
+    host: {{ external_database.clair.host }}
+    port: {{ external_database.clair.port }}
+    db_name: {{ external_database.clair.db_name }}
+    username: {{ external_database.clair.username }}
+    password: {{ external_database.clair.password }}
+    ssl_mode: {{ external_database.clair.ssl_mode }}
+  notary_signer:
+    host: {{ external_database.notary_signer.host }}
+    port: {{ external_database.notary_signer.port }}
+    db_name: {{external_database.notary_signer.db_name }}
+    username: {{ external_database.notary_signer.username }}
+    password: {{ external_database.notary_signer.password }}
+    ssl_mode: {{ external_database.notary_signer.ssl_mode }}
+  notary_server:
+    host: {{ external_database.notary_server.host }}
+    port: {{ external_database.notary_server.port }}
+    db_name: {{ external_database.notary_server.db_name }}
+    username: {{ external_database.notary_server.username }}
+    password: {{ external_database.notary_server.password }}
+    ssl_mode: {{ external_database.notary_server.ssl_mode }}
+{% else %}
+# Uncomment external_database if using external database.
+# external_database:
+#   harbor:
+#     host: harbor_db_host
+#     port: harbor_db_port
+#     db_name: harbor_db_name
+#     username: harbor_db_username
+#     password: harbor_db_password
+#     ssl_mode: disable
+#   clair:
+#     host: clair_db_host
+#     port: clair_db_port
+#     db_name: clair_db_name
+#     username: clair_db_username
+#     password: clair_db_password
+#     ssl_mode: disable
+#   notary_signer:
+#     host: notary_signer_db_host
+#     port: notary_signer_db_port
+#     db_name: notary_signer_db_name
+#     username: notary_signer_db_username
+#     password: notary_signer_db_password
+#     ssl_mode: disable
+#   notary_server:
+#     host: notary_server_db_host
+#     port: notary_server_db_port
+#     db_name: notary_server_db_name
+#     username: notary_server_db_username
+#     password: notary_server_db_password
+#     ssl_mode: disable
+{% endif %}
+
+{% if external_redis is defined %}
+external_redis:
+  host: {{ external_redis.host }}
+  port: {{ external_redis.port }}
+  password: {{ external_redis.password }}
+  # db_index 0 is for core, it's unchangeable
+  registry_db_index: {{ external_redis.registry_db_index }}
+  jobservice_db_index: {{ external_redis.jobservice_db_index }}
+  chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }}
+  clair_db_index: {{ external_redis.clair_db_index }}
+  trivy_db_index: {{ external_redis.trivy_db_index }}
+  idle_timeout_seconds: {{ external_redis.idle_timeout_seconds }}
+{% else %}
+# Umcomments external_redis if using external Redis server
+# external_redis:
+#   host: redis
+#   port: 6379
+#   password:
+#   # db_index 0 is for core, it's unchangeable
+#   registry_db_index: 1
+#   jobservice_db_index: 2
+#   chartmuseum_db_index: 3
+#   clair_db_index: 4
+#   trivy_db_index: 5
+#   idle_timeout_seconds: 30
+{% endif %}
+
+{% if uaa is defined %}
+# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
+uaa:
+  ca_file: {{ uaa.ca_file }}
+{% else %}
+# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
+# uaa:
+#   ca_file: /path/to/ca
+{% endif %}
+
+
+# Global proxy
+# Config http proxy for components, e.g. http://my.proxy.com:3128
+# Components doesn't need to connect to each others via http proxy.
+# Remove component from `components` array if want disable proxy
+# for it. If you want use proxy for replication, MUST enable proxy
+# for core and jobservice, and set `http_proxy` and `https_proxy`.
+# Add domain to the `no_proxy` field, when you want disable proxy
+# for some special registry.
+{% if proxy is defined %}
+proxy:
+  http_proxy: {{ proxy.http_proxy or ''}}
+  https_proxy: {{ proxy.https_proxy or ''}}
+  no_proxy: {{ proxy.no_proxy or ''}}
+  {% if proxy.components is defined %}
+  components:
+    {% for component in proxy.components %}
+    - {{component}}
+    {% endfor %}
+    - trivy
+  {% endif %}
+{% else %}
+proxy:
+  http_proxy:
+  https_proxy:
+  no_proxy:
+  components:
+    - core
+    - jobservice
+    - clair
+    - trivy
+{% endif %}