diff --git a/tools/migrate_chart/Dockerfile b/tools/migrate_chart/Dockerfile new file mode 100644 index 000000000..877531b87 --- /dev/null +++ b/tools/migrate_chart/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.8.5-slim + +ENV HELM_EXPERIMENTAL_OCI=1 +ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + +COPY ./migrate_chart.py ./migrate_chart.sh / +ADD https://get.helm.sh/helm-v3.2.4-linux-amd64.tar.gz / + +RUN tar zxvf /helm-v3.2.4-linux-amd64.tar.gz && \ + pip install click==7.1.2 && \ + pip install requests==2.24.0 && \ + chmod +x /migrate_chart.sh ./migrate_chart.py + +ENTRYPOINT [ "/migrate_chart.py" ] \ No newline at end of file diff --git a/tools/migrate_chart/Readme.md b/tools/migrate_chart/Readme.md new file mode 100644 index 000000000..dc5a8d28b --- /dev/null +++ b/tools/migrate_chart/Readme.md @@ -0,0 +1,36 @@ +# Chart Migrating Tool + +Harbor supports two different ways to storage the chart data. + + 1. stored in Harbor registry storage directly via OCI API. + 2. stored in Harbor hosted chartmuseum backend via chartmuseam's API + +There is an performance issue in chartmuseam. For example, on my 2 core 8G memory test environment, to get 10000 charts infomation needs about 10s to return, In 50000 charts situation, It will cause a timeout error. + +After version 2.0, Harbor becomes to the OCI registry. So It can storage chart content directly without chartmuseum. + +This tool used to migrate the legacy helm charts stored in the chartmuseum backend to Harbor OCI registry backend. + +On test environment( 2 core 8G memory), using this tool to migrate 10000 charts needs about 2~3 hours + +## Usages + +Compile the chart with command + +``` sh +docker build -t goharbor/migrate-chart:0.1.0 . +``` + +Migrate charts run command below: + +``` sh +docker run -it --rm -v {{your_chart_data_location}}:/chart_storage -v {{harbor_ca_cert_location}}:/usr/local/share/ca-certificates/harbor_ca.crt goharbor/migrate-chart:0.1.0 --hostname {{harbor_hostname}} --password {{harbor_admin_password}} +``` + +* `your_chart_data_location`: The location of your chart storage chart. By default, it's the `chart_storage` dir inside Harbor's `data_volumn` + +* `harbor_ca_cert_location`: If Harbor enabled HTTPS, you need to add the `ca_cert` for connecting Harbor + +* `harbor_hostname`: The hostname of Harbor + +* `harbor_admin_password`: The password of harbor admin user diff --git a/tools/migrate_chart/migrate_chart.py b/tools/migrate_chart/migrate_chart.py new file mode 100644 index 000000000..6efb766e1 --- /dev/null +++ b/tools/migrate_chart/migrate_chart.py @@ -0,0 +1,107 @@ +#!/usr/local/bin/python3 + +import subprocess +import signal +import sys +from pathlib import Path + +import click +import requests + +MIGRATE_CHART_SCRIPT = '/migrate_chart.sh' +HELM_CMD = '/linux-amd64/helm' +CA_UPDATE_CMD = 'update-ca-certificates' +CHART_URL_PATTERN = "https://{host}/api/v2.0/projects/{project}/repositories/{name}/artifacts/{version}" +CHART_SOURCE_DIR = Path('/chart_storage') + +errs = [] + +def print_exist_errs(): + if errs: + click.echo("Following errors exist", err=True) + for e in errs: + click.echo(e, err=True) + +def graceful_exit(signum, frame): + print_exist_errs() + sys.exit() + +signal.signal(signal.SIGINT, graceful_exit) +signal.signal(signal.SIGTERM, graceful_exit) + +class ChartV2: + + def __init__(self, filepath:Path): + self.filepath = filepath + self.project = self.filepath.parts[-2] + parts = self.filepath.stem.split('-') + flag = False + for i in range(len(parts)-1, -1, -1): + if parts[i][0].isnumeric(): + self.name, self.version = '-'.join(parts[:i]), '-'.join(parts[i:]) + flag = True + break + if not flag: + raise Exception('chart name: {} is illegal'.format('-'.join(parts))) + + def __check_exist(self, hostname, username, password): + return requests.get(CHART_URL_PATTERN.format( + host=hostname, + project=self.project, + name=self.name, + version=self.version), + auth=requests.auth.HTTPBasicAuth(username, password)) + + def migrate(self, hostname, username, password): + res = self.__check_exist(hostname, username, password) + if res.status_code == 200: + raise Exception("Artifact already exist in harbor") + if res.status_code == 401: + raise Exception(res.reason) + + oci_ref = "{host}/{project}/{name}:{version}".format( + host=hostname, + project=self.project, + name=self.name, + version=self.version) + + return subprocess.run([MIGRATE_CHART_SCRIPT, HELM_CMD, self.filepath, oci_ref], + text=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + + +@click.command() +@click.option('--hostname', default='127.0.0.1', help='the password to login harbor') +@click.option('--username', default='admin', help='The username to login harbor') +@click.option('--password', default='Harbor12345', help='the password to login harbor') +def migrate(hostname, username, password): + """ + Migrate chart v2 to harbor oci registry + """ + if username != 'admin': + raise Exception('This operation only allowed for admin') + subprocess.run([CA_UPDATE_CMD]) + subprocess.run([HELM_CMD, 'registry', 'login', hostname, '--username', username, '--password', password]) + charts = [ChartV2(c) for p in CHART_SOURCE_DIR.iterdir() if p.is_dir() for c in p.iterdir() if c.is_file() and c.name != "index-cache.yaml"] + with click.progressbar(charts, label="Migrating chart ...", length=len(charts), + item_show_func=lambda x: "{}/{}:{} total errors: {}".format(x.project, x.name, x.version, len(errs)) if x else '') as bar: + for chart in bar: + try: + result = chart.migrate(hostname, username, password) + if result.stderr: + errs.append("chart: {name}:{version} in {project} has err: {err}".format( + name=chart.name, + version=chart.version, + project=chart.project, + err=result.stderr + )) + except Exception as e: + errs.append("chart: {name}:{version} in {project} has err: {err}".format( + name=chart.name, + version=chart.version, + project=chart.project, + err=e)) + click.echo("Migration is Done.") + print_exist_errs() + +if __name__ == '__main__': + migrate() diff --git a/tools/migrate_chart/migrate_chart.sh b/tools/migrate_chart/migrate_chart.sh new file mode 100644 index 000000000..a6fd5f791 --- /dev/null +++ b/tools/migrate_chart/migrate_chart.sh @@ -0,0 +1,9 @@ +#! /bin/bash + +HELM_CMD=$1 +V2_CHART_PATH=$2 +OCI_REF=$3 + +${HELM_CMD} chart save ${V2_CHART_PATH} ${OCI_REF} +${HELM_CMD} chart push ${OCI_REF} +${HELM_CMD} chart remove ${OCI_REF}