Add tool for migration chart v2 to oci format

chartmuseum might deprecated in future. This tool is used for migrate the charts

Signed-off-by: DQ <dengq@vmware.com>

tools/migrate_chart/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.8.5-slim
+
+ENV HELM_EXPERIMENTAL_OCI=1
+ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+
+COPY ./migrate_chart.py ./migrate_chart.sh /
+ADD https://get.helm.sh/helm-v3.2.4-linux-amd64.tar.gz /
+
+RUN tar zxvf /helm-v3.2.4-linux-amd64.tar.gz && \
+    pip install click==7.1.2 && \
+    pip install requests==2.24.0 && \
+    chmod +x /migrate_chart.sh ./migrate_chart.py
+
+ENTRYPOINT [ "/migrate_chart.py" ]

tools/migrate_chart/Readme.md

@@ -0,0 +1,36 @@
+# Chart Migrating Tool
+
+Harbor supports two different ways to storage the chart data.
+
+   1. stored in Harbor registry storage directly via OCI API.
+   2. stored in Harbor hosted chartmuseum backend via chartmuseam's API
+
+There is an performance issue in chartmuseam. For example, on my 2 core 8G memory test environment, to get 10000 charts infomation needs about 10s to return, In 50000 charts situation, It will cause a timeout error.
+
+After version 2.0, Harbor becomes to the OCI registry. So It can storage chart content directly without chartmuseum.
+
+This tool used to migrate the legacy helm charts stored in the chartmuseum backend to Harbor OCI registry backend.
+
+On test environment( 2 core 8G memory), using this tool to migrate 10000 charts needs about 2~3 hours
+
+## Usages
+
+Compile the chart with command
+
+``` sh
+docker build -t goharbor/migrate-chart:0.1.0 .
+```
+
+Migrate charts run command below:
+
+``` sh
+docker run -it --rm -v {{your_chart_data_location}}:/chart_storage -v {{harbor_ca_cert_location}}:/usr/local/share/ca-certificates/harbor_ca.crt  goharbor/migrate-chart:0.1.0 --hostname {{harbor_hostname}} --password {{harbor_admin_password}}
+```
+
+* `your_chart_data_location`: The location of your chart storage chart. By default, it's the `chart_storage` dir inside Harbor's `data_volumn`
+
+* `harbor_ca_cert_location`: If Harbor enabled HTTPS, you need to add the `ca_cert` for connecting Harbor
+
+* `harbor_hostname`: The hostname of Harbor
+
+* `harbor_admin_password`: The password of harbor admin user

tools/migrate_chart/migrate_chart.py

@@ -0,0 +1,107 @@
+#!/usr/local/bin/python3
+
+import subprocess
+import signal
+import sys
+from pathlib import Path
+
+import click
+import requests
+
+MIGRATE_CHART_SCRIPT = '/migrate_chart.sh'
+HELM_CMD = '/linux-amd64/helm'
+CA_UPDATE_CMD = 'update-ca-certificates'
+CHART_URL_PATTERN = "https://{host}/api/v2.0/projects/{project}/repositories/{name}/artifacts/{version}"
+CHART_SOURCE_DIR = Path('/chart_storage')
+
+errs = []
+
+def print_exist_errs():
+    if errs:
+        click.echo("Following errors exist", err=True)
+        for e in errs:
+            click.echo(e, err=True)
+
+def graceful_exit(signum, frame):
+    print_exist_errs()
+    sys.exit()
+
+signal.signal(signal.SIGINT, graceful_exit)
+signal.signal(signal.SIGTERM, graceful_exit)
+
+class ChartV2:
+
+    def __init__(self, filepath:Path):
+        self.filepath = filepath
+        self.project = self.filepath.parts[-2]
+        parts = self.filepath.stem.split('-')
+        flag = False
+        for i in range(len(parts)-1, -1, -1):
+            if parts[i][0].isnumeric():
+                self.name, self.version = '-'.join(parts[:i]), '-'.join(parts[i:])
+                flag = True
+                break
+        if not flag:
+            raise Exception('chart name: {} is illegal'.format('-'.join(parts)))
+
+    def __check_exist(self, hostname, username, password):
+        return requests.get(CHART_URL_PATTERN.format(
+                host=hostname,
+                project=self.project,
+                name=self.name,
+                version=self.version),
+                auth=requests.auth.HTTPBasicAuth(username, password))
+
+    def migrate(self, hostname, username, password):
+        res = self.__check_exist(hostname, username, password)
+        if res.status_code == 200:
+            raise Exception("Artifact already exist in harbor")
+        if res.status_code == 401:
+            raise Exception(res.reason)
+
+        oci_ref = "{host}/{project}/{name}:{version}".format(
+            host=hostname,
+            project=self.project,
+            name=self.name,
+            version=self.version)
+
+        return subprocess.run([MIGRATE_CHART_SCRIPT, HELM_CMD, self.filepath, oci_ref],
+        text=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+
+
+@click.command()
+@click.option('--hostname', default='127.0.0.1', help='the password to login harbor')
+@click.option('--username', default='admin', help='The username to login harbor')
+@click.option('--password', default='Harbor12345', help='the password to login harbor')
+def migrate(hostname, username, password):
+    """
+    Migrate chart v2 to harbor oci registry
+    """
+    if username != 'admin':
+        raise Exception('This operation only allowed for admin')
+    subprocess.run([CA_UPDATE_CMD])
+    subprocess.run([HELM_CMD, 'registry', 'login', hostname, '--username', username, '--password', password])
+    charts = [ChartV2(c) for p in CHART_SOURCE_DIR.iterdir() if p.is_dir() for c in p.iterdir() if c.is_file() and c.name != "index-cache.yaml"]
+    with click.progressbar(charts, label="Migrating chart ...", length=len(charts),
+    item_show_func=lambda x: "{}/{}:{} total errors: {}".format(x.project, x.name, x.version, len(errs)) if x else '') as bar:
+        for chart in bar:
+            try:
+                result = chart.migrate(hostname, username, password)
+                if result.stderr:
+                    errs.append("chart: {name}:{version} in {project} has err: {err}".format(
+                        name=chart.name,
+                        version=chart.version,
+                        project=chart.project,
+                        err=result.stderr
+                    ))
+            except Exception as e:
+                errs.append("chart: {name}:{version} in {project} has err: {err}".format(
+                    name=chart.name,
+                    version=chart.version,
+                    project=chart.project,
+                    err=e))
+    click.echo("Migration is Done.")
+    print_exist_errs()
+
+if __name__ == '__main__':
+    migrate()

tools/migrate_chart/migrate_chart.sh

@@ -0,0 +1,9 @@
+#! /bin/bash
+
+HELM_CMD=$1
+V2_CHART_PATH=$2
+OCI_REF=$3
+
+${HELM_CMD} chart save ${V2_CHART_PATH} ${OCI_REF}
+${HELM_CMD} chart push ${OCI_REF}
+${HELM_CMD} chart remove ${OCI_REF}