From 4617e0ff3841991fc1ee4f092e9c3b3df50c25cd Mon Sep 17 00:00:00 2001 From: DQ Date: Sun, 28 Jun 2020 17:49:14 +0800 Subject: [PATCH] Enhance: Support multi downversion in migration 1. Change down version to list to accept multi verstion value 2. Update search function use BFS to find migration path 2. Add test case Signed-off-by: DQ --- make/photon/prepare/commands/migrate.py | 3 + .../migrations/version_1_10_0/__init__.py | 2 +- .../migrations/version_1_9_0/__init__.py | 6 +- .../migrations/version_1_9_0/harbor.yml.jinja | 2 +- .../migrations/version_2_0_0/__init__.py | 2 +- .../prepare/tests/migrations/utils_test.py | 60 ++++++++++----- make/photon/prepare/utils/migration.py | 75 ++++++++++++------- 7 files changed, 98 insertions(+), 52 deletions(-) diff --git a/make/photon/prepare/commands/migrate.py b/make/photon/prepare/commands/migrate.py index 3abc0c8a8..c6fd0d34b 100644 --- a/make/photon/prepare/commands/migrate.py +++ b/make/photon/prepare/commands/migrate.py @@ -14,6 +14,9 @@ from migrations import accept_versions def migrate(input_, output, target): """ migrate command will migrate config file style to specific version + :input_: is the path of the original config file + :output: is the destination path of config file, the generated configs will storage in it + :target: is the the target version of config file will upgrade to """ if target not in accept_versions: click.echo('target version {} not supported'.format(target)) diff --git a/make/photon/prepare/migrations/version_1_10_0/__init__.py b/make/photon/prepare/migrations/version_1_10_0/__init__.py index 4d383c937..84b89c0a1 100644 --- a/make/photon/prepare/migrations/version_1_10_0/__init__.py +++ b/make/photon/prepare/migrations/version_1_10_0/__init__.py @@ -3,7 +3,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined from utils.migration import read_conf revision = '1.10.0' -down_revision = '1.9.0' +down_revisions = ['1.9.0'] def migrate(input_cfg, output_cfg): config_dict = read_conf(input_cfg) diff --git a/make/photon/prepare/migrations/version_1_9_0/__init__.py b/make/photon/prepare/migrations/version_1_9_0/__init__.py index 041dcf4f7..1e9cdd61f 100644 --- a/make/photon/prepare/migrations/version_1_9_0/__init__.py +++ b/make/photon/prepare/migrations/version_1_9_0/__init__.py @@ -3,14 +3,14 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined from utils.migration import read_conf revision = '1.9.0' -down_revision = None +down_revisions = [] def migrate(input_cfg, output_cfg): config_dict = read_conf(input_cfg) - this_dir = os.path.dirname(__file__) + current_dir = os.path.dirname(__file__) tpl = Environment( - loader=FileSystemLoader(this_dir), + loader=FileSystemLoader(current_dir), undefined=StrictUndefined, trim_blocks=True, lstrip_blocks=True diff --git a/make/photon/prepare/migrations/version_1_9_0/harbor.yml.jinja b/make/photon/prepare/migrations/version_1_9_0/harbor.yml.jinja index c8867486a..6d6b9750b 100644 --- a/make/photon/prepare/migrations/version_1_9_0/harbor.yml.jinja +++ b/make/photon/prepare/migrations/version_1_9_0/harbor.yml.jinja @@ -48,7 +48,7 @@ harbor_admin_password: {{ harbor_admin_password }} # Harbor DB configuration database: # The password for the root user of Harbor DB. Change this before any production use. - password: {{ database.password}} + password: {{ database.password }} # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained. max_idle_conns: 50 # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections. diff --git a/make/photon/prepare/migrations/version_2_0_0/__init__.py b/make/photon/prepare/migrations/version_2_0_0/__init__.py index 344ce5b63..9dd4c135d 100644 --- a/make/photon/prepare/migrations/version_2_0_0/__init__.py +++ b/make/photon/prepare/migrations/version_2_0_0/__init__.py @@ -3,7 +3,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined from utils.migration import read_conf revision = '2.0.0' -down_revision = '1.10.0' +down_revisions = ['1.10.0'] def migrate(input_cfg, output_cfg): config_dict = read_conf(input_cfg) diff --git a/make/photon/prepare/tests/migrations/utils_test.py b/make/photon/prepare/tests/migrations/utils_test.py index 5fd15b5c6..df74c5399 100644 --- a/make/photon/prepare/tests/migrations/utils_test.py +++ b/make/photon/prepare/tests/migrations/utils_test.py @@ -1,36 +1,46 @@ import pytest import importlib -from utils.migration import search +from utils.migration import search, MigratioNotFound class mockModule: - def __init__(self, revision, down_revision): + def __init__(self, revision: str, down_revisions: list): self.revision = revision - self.down_revision = down_revision + self.down_revisions = down_revisions def mock_import_module_loop(module_path: str): - loop_modules = { - 'migration.versions.1_9_0': mockModule('1.9.0', None), - 'migration.versions.1_10_0': mockModule('1.10.0', '2.0.0'), - 'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0') + modules = { + 'migrations.version_1_9_0': mockModule('1.9.0', []), + 'migrations.version_1_10_0': mockModule('1.10.0', ['2.0.0']), + 'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0']) } - return loop_modules[module_path] + return modules[module_path] def mock_import_module_mission(module_path: str): - loop_modules = { - 'migration.versions.1_9_0': mockModule('1.9.0', None), - 'migration.versions.1_10_0': mockModule('1.10.0', None), - 'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0') + modules = { + 'migrations.version_1_9_0': mockModule('1.9.0', []), + 'migrations.version_1_10_0': mockModule('1.10.0', []), + 'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0']) } - return loop_modules[module_path] + return modules[module_path] def mock_import_module_success(module_path: str): - loop_modules = { - 'migration.versions.1_9_0': mockModule('1.9.0', None), - 'migration.versions.1_10_0': mockModule('1.10.0', '1.9.0'), - 'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0') + modules = { + 'migrations.version_1_9_0': mockModule('1.9.0', []), + 'migrations.version_1_10_0': mockModule('1.10.0', ['1.9.0']), + 'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0']) } - return loop_modules[module_path] + return modules[module_path] + +def mock_import_module_success_multi_downversion(module_path: str): + modules = { + 'migrations.version_1_9_0': mockModule('1.9.0', []), + 'migrations.version_1_10_0': mockModule('1.10.0', ['1.9.0']), + 'migrations.version_1_10_1': mockModule('1.10.1', ['1.9.0']), + 'migrations.version_1_10_2': mockModule('1.10.2', ['1.9.0']), + 'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0', '1.10.1', '1.10.2']) + } + return modules[module_path] @pytest.fixture def mock_import_module_with_loop(monkeypatch): @@ -44,15 +54,25 @@ def mock_import_module_with_mission(monkeypatch): def mock_import_module_with_success(monkeypatch): monkeypatch.setattr(importlib, "import_module", mock_import_module_success) +@pytest.fixture +def mock_import_module_with_success_multi_downversion(monkeypatch): + monkeypatch.setattr(importlib, "import_module", mock_import_module_success_multi_downversion) + def test_search_loop(mock_import_module_with_loop): with pytest.raises(Exception): search('1.9.0', '2.0.0') def test_search_mission(mock_import_module_with_mission): - with pytest.raises(Exception): + with pytest.raises(MigratioNotFound): search('1.9.0', '2.0.0') -def test_search_success(): +def test_search_success(mock_import_module_with_success): migration_path = search('1.9.0', '2.0.0') assert migration_path[0].revision == '1.10.0' assert migration_path[1].revision == '2.0.0' + +def test_search_success_multi_downversion(mock_import_module_with_success_multi_downversion): + migration_path = search('1.9.0', '2.0.0') + print(migration_path) + assert migration_path[0].revision == '1.10.2' + assert migration_path[1].revision == '2.0.0' diff --git a/make/photon/prepare/utils/migration.py b/make/photon/prepare/utils/migration.py index 778b7e496..1389bae45 100644 --- a/make/photon/prepare/utils/migration.py +++ b/make/photon/prepare/utils/migration.py @@ -4,7 +4,24 @@ import importlib import os from collections import deque -from migrations import MIGRATION_BASE_DIR +class MigratioNotFound(Exception): ... + +class MigrationVersion: + ''' + The version used to migration + + Arttribute: + name(str): version name like `1.0.0` + module: the python module object for a specific migration which contains migrate info, codes and templates + down_versions(list): previous versions that can migrated to this version + ''' + def __init__(self, version: str): + self.name = version + self.module = importlib.import_module("migrations.version_{}".format(version.replace(".","_"))) + + @property + def down_versions(self): + return self.module.down_revisions def read_conf(path): with open(path) as f: @@ -15,29 +32,35 @@ def read_conf(path): exit(-1) return d -def _to_module_path(ver): - return "migrations.version_{}".format(ver.replace(".","_")) +def search(input_version: str, target_version: str) -> list : + """ + Find the migration path by BFS + Args: + input_version(str): The version migration start from + target_version(str): The target version migrated to + Returns: + list: the module of migrations in the upgrade path + """ + upgrade_path = [] + next_version, visited, q = {}, set(), deque() + q.append(target_version) + found = False + while q: # BFS to find a valid path + version = MigrationVersion(q.popleft()) + visited.add(version.name) + if version.name == input_version: + found = True + break # break loop cause migration path found + for v in version.down_versions: + next_version[v] = version.name + if v not in (visited.union(q)): + q.append(v) -def search(input_ver: str, target_ver: str) -> deque : - """ - Search accept a input version and the target version. - Returns the module of migrations in the upgrade path - """ - upgrade_path, visited = deque(), set() - while True: - module_path = _to_module_path(target_ver) - visited.add(target_ver) # mark current version for loop finding - if os.path.isdir(os.path.join(MIGRATION_BASE_DIR, 'version_{}'.format(target_ver.replace(".","_")))): - module = importlib.import_module(module_path) - if module.revision == input_ver: # migration path found - break - elif module.down_revision is None: # migration path not found - raise Exception('no migration path found') - else: - upgrade_path.appendleft(module) - target_ver = module.down_revision - if target_ver in visited: # version visited before, loop found - raise Exception('find a loop caused by {} on migration path'.format(target_ver)) - else: - raise Exception('{} not dir'.format(os.path.join(MIGRATION_BASE_DIR, 'versions', target_ver.replace(".","_")))) - return upgrade_path + if not found: + raise MigratioNotFound('no migration path found to target version') + + current_version = MigrationVersion(input_version) + while current_version.name != target_version: + current_version = MigrationVersion(next_version[current_version.name]) + upgrade_path.append(current_version) + return list(map(lambda x: x.module, upgrade_path)) \ No newline at end of file