From 950fc06a87abe66bde7730bf481e78c45b6638d6 Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Wed, 6 Mar 2024 02:51:38 +0800 Subject: [PATCH] Update migration tools (#20075) update migration tools Signed-off-by: stonezdj Co-authored-by: stonezdj --- tools/migrate_chart/Dockerfile | 1 + tools/migrate_chart/migrate_chart.py | 52 +++++++++++++++++++---- tools/migrate_chart/test_migrate_chart.py | 45 ++++++++++++++++++++ 3 files changed, 89 insertions(+), 9 deletions(-) create mode 100644 tools/migrate_chart/test_migrate_chart.py diff --git a/tools/migrate_chart/Dockerfile b/tools/migrate_chart/Dockerfile index 53c1183c5..91f9c813b 100644 --- a/tools/migrate_chart/Dockerfile +++ b/tools/migrate_chart/Dockerfile @@ -9,6 +9,7 @@ ADD https://get.helm.sh/helm-v3.9.1-linux-amd64.tar.gz / RUN tar zxvf /helm-v3.9.1-linux-amd64.tar.gz && \ pip install click==7.1.2 && \ pip install requests==2.24.0 && \ + pip install pyyaml && \ chmod +x /migrate_chart.sh ./migrate_chart.py ENTRYPOINT [ "/migrate_chart.py" ] \ No newline at end of file diff --git a/tools/migrate_chart/migrate_chart.py b/tools/migrate_chart/migrate_chart.py index 70a1f00cd..71bfc4470 100644 --- a/tools/migrate_chart/migrate_chart.py +++ b/tools/migrate_chart/migrate_chart.py @@ -3,7 +3,10 @@ import subprocess import signal import sys +import os from pathlib import Path +import tarfile +import yaml import click import requests @@ -32,24 +35,51 @@ def graceful_exit(signum, frame): signal.signal(signal.SIGINT, graceful_exit) signal.signal(signal.SIGTERM, graceful_exit) +def find_chart_yaml(tar, path=''): + # Iterate through the members of the tarfile + for member in tar.getmembers(): + # If the member is a directory, recursively search within it + if member.isdir(): + find_chart_yaml(tar, os.path.join(path, member.name)) + # If the member is a file and its name is 'chart.yaml', return its path + if "Chart.yaml" in member.name: + return os.path.join(path, member.name) + +def read_chart_version(chart_tgz_path): + # Open the chart tgz file + with tarfile.open(chart_tgz_path, 'r:gz') as tar: + # Find the path to chart.yaml within the tarball + chart_yaml_path = find_chart_yaml(tar) + if chart_yaml_path: + # Extract the chart.yaml file + chart_yaml_file = tar.extractfile(chart_yaml_path) + if chart_yaml_file is not None: + # Load the YAML content from chart.yaml + chart_data = yaml.safe_load(chart_yaml_file) + # Read the version from chart.yaml + version = chart_data.get('version') + name = chart_data.get('name') + return name, version + else: + raise Exception("Failed to read chart.yaml from the chart tgz file. filename {}".format(chart_tgz_path)) + else: + raise Exception("chart.yaml not found in the chart tgz file. filename {}".format(chart_tgz_path)) + class ChartV2: def __init__(self, filepath:Path): self.filepath = filepath self.project = self.filepath.parts[-2] - parts = self.filepath.stem.split('-') - flag = False + self.name = "" + self.version = "" try: - for i in range(len(parts)-1, -1, -1): - if parts[i][0].isnumeric() or ((parts[i][0]=='v' or parts[i][0]=='v') and parts[i][1].isnumeric()) : - self.name, self.version = '-'.join(parts[:i]), '-'.join(parts[i:]) - flag = True - break - if not flag: + self.name, self.version = read_chart_version(filepath) + if self.name == "" or self.version == "" or self.name is None or self.version is None : raise Exception('chart name: {} is illegal'.format('-'.join(parts))) except Exception as e: click.echo("Skipped chart: {} due to illegal chart name. Error: {}".format(filepath, e), err=True) return + def __check_exist(self, hostname, username, password): return requests.get(CHART_URL_PATTERN.format( host=hostname, @@ -90,6 +120,9 @@ def migrate(hostname, username, password): item_show_func=lambda x: "{}/{}:{} total errors: {}".format(x.project, x.name, x.version, len(errs)) if x else '') as bar: for chart in bar: try: + if chart.name == "" or chart.version == "" : + print("skip the chart {} has no name or version info".format(chart.filepath)) + continue result = chart.migrate(hostname, username, password) if result.stderr: errs.append("chart: {name}:{version} in {project} has err: {err}".format( @@ -99,10 +132,11 @@ def migrate(hostname, username, password): err=result.stderr )) except Exception as e: - errs.append("chart: {name}:{version} in {project} has err: {err}".format( + errs.append("chart: {name}:{version} in {project}, path {path} has err: {err}".format( name=chart.name, version=chart.version, project=chart.project, + path = chart.filepath, err=e)) click.echo("Migration is Done.") print_exist_errs() diff --git a/tools/migrate_chart/test_migrate_chart.py b/tools/migrate_chart/test_migrate_chart.py new file mode 100644 index 000000000..b3d5e01ac --- /dev/null +++ b/tools/migrate_chart/test_migrate_chart.py @@ -0,0 +1,45 @@ + +import unittest +import semver +import tempfile +import os +import tarfile +import shutil +from pathlib import Path +from migrate_chart import extract_chart_name_and_version +from migrate_chart import read_chart_version + +class TestExtractChartNameAndVersion(unittest.TestCase): + def test_valid_chart_name(self): + filepath = Path("my-project/my-chart-v1.0.0.tgz") + name, version = extract_chart_name_and_version(filepath) + self.assertEqual(name, "my-chart") + self.assertEqual(version, "v1.0.0") + + def test_invalid_chart_name(self): + filepath = Path("my-project/mychart.tgz") + name, version = extract_chart_name_and_version(filepath) + self.assertIsNone(name) + self.assertIsNone(version) + + # def test_pure_digit(self): + # filepath = Path("my-project/my-chart-8.0.0-5.tgz") + # name, version = extract_chart_name_and_version(filepath) + # self.assertEqual(name, "my-chart") + # self.assertEqual(version, "8.0.0") + + # def test_digit_startv(self): + # filepath = Path("my-project/my-chart-v8.0.0-5.tgz") + # name, version = extract_chart_name_and_version(filepath) + # self.assertEqual(name, "my-chart") + # self.assertEqual(version, "8.0.0") + + def test_parse_version(self): + temp_dir = tempfile.mkdtemp() + file_name = "/Users/daojunz/Downloads/cert-manager/sample/cert-manager-8.0.0-5.tgz" + name, version = read_chart_version(file_name) + print(name) + print(version) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file