Merge pull request #12341 from ninjadq/support_multi_down_version

Enhance: Support multi downversion in migration

make/photon/prepare/commands/migrate.py

@@ -14,6 +14,9 @@ from migrations import accept_versions
 def migrate(input_, output, target):
     """
     migrate command will migrate config file style to specific version
+    :input_: is the path of the original config file
+    :output: is the destination path of config file, the generated configs will storage in it
+    :target: is the the target version of config file will upgrade to
     """
     if target not in accept_versions:
         click.echo('target version {} not supported'.format(target))

make/photon/prepare/migrations/version_1_10_0/__init__.py

@@ -3,7 +3,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined
 from utils.migration import read_conf
 
 revision = '1.10.0'
-down_revision = '1.9.0'
+down_revisions = ['1.9.0']
 
 def migrate(input_cfg, output_cfg):
     config_dict = read_conf(input_cfg)

make/photon/prepare/migrations/version_1_9_0/__init__.py

@@ -3,14 +3,14 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined
 from utils.migration import read_conf
 
 revision = '1.9.0'
-down_revision = None
+down_revisions = []
 
 def migrate(input_cfg, output_cfg):
     config_dict = read_conf(input_cfg)
 
-    this_dir = os.path.dirname(__file__)
+    current_dir = os.path.dirname(__file__)
     tpl = Environment(
-        loader=FileSystemLoader(this_dir),
+        loader=FileSystemLoader(current_dir),
         undefined=StrictUndefined,
         trim_blocks=True,
         lstrip_blocks=True

make/photon/prepare/migrations/version_1_9_0/harbor.yml.jinja

@@ -48,7 +48,7 @@ harbor_admin_password: {{ harbor_admin_password }}
 # Harbor DB configuration
 database:
   # The password for the root user of Harbor DB. Change this before any production use.
-  password: {{ database.password}}
+  password: {{ database.password }}
   # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
   max_idle_conns: 50
   # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.

make/photon/prepare/migrations/version_2_0_0/__init__.py

@@ -3,7 +3,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined
 from utils.migration import read_conf
 
 revision = '2.0.0'
-down_revision = '1.10.0'
+down_revisions = ['1.10.0']
 
 def migrate(input_cfg, output_cfg):
     config_dict = read_conf(input_cfg)

make/photon/prepare/tests/migrations/utils_test.py

@@ -1,36 +1,46 @@
 import pytest
 import importlib
 
-from utils.migration import search
+from utils.migration import search, MigratioNotFound
 
 class mockModule:
-    def __init__(self, revision, down_revision):
+    def __init__(self, revision: str, down_revisions: list):
         self.revision = revision
-        self.down_revision = down_revision
+        self.down_revisions = down_revisions
 
 def mock_import_module_loop(module_path: str):
-    loop_modules = {
+    modules = {
-    'migration.versions.1_9_0': mockModule('1.9.0', None),
+    'migrations.version_1_9_0': mockModule('1.9.0', []),
-    'migration.versions.1_10_0': mockModule('1.10.0', '2.0.0'),
+    'migrations.version_1_10_0': mockModule('1.10.0', ['2.0.0']),
-    'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0')
+    'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0'])
     }
-    return loop_modules[module_path]
+    return modules[module_path]
 
 def mock_import_module_mission(module_path: str):
-    loop_modules = {
+    modules = {
-    'migration.versions.1_9_0': mockModule('1.9.0', None),
+    'migrations.version_1_9_0': mockModule('1.9.0', []),
-    'migration.versions.1_10_0': mockModule('1.10.0', None),
+    'migrations.version_1_10_0': mockModule('1.10.0', []),
-    'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0')
+    'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0'])
     }
-    return loop_modules[module_path]
+    return modules[module_path]
 
 def mock_import_module_success(module_path: str):
-    loop_modules = {
+    modules = {
-    'migration.versions.1_9_0': mockModule('1.9.0', None),
+    'migrations.version_1_9_0': mockModule('1.9.0', []),
-    'migration.versions.1_10_0': mockModule('1.10.0', '1.9.0'),
+    'migrations.version_1_10_0': mockModule('1.10.0', ['1.9.0']),
-    'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0')
+    'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0'])
     }
-    return loop_modules[module_path]
+    return modules[module_path]
+
+def mock_import_module_success_multi_downversion(module_path: str):
+    modules = {
+    'migrations.version_1_9_0': mockModule('1.9.0', []),
+    'migrations.version_1_10_0': mockModule('1.10.0', ['1.9.0']),
+    'migrations.version_1_10_1': mockModule('1.10.1', ['1.9.0']),
+    'migrations.version_1_10_2': mockModule('1.10.2', ['1.9.0']),
+    'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0', '1.10.1', '1.10.2'])
+    }
+    return modules[module_path]
 
 @pytest.fixture
 def mock_import_module_with_loop(monkeypatch):
@@ -44,15 +54,25 @@ def mock_import_module_with_mission(monkeypatch):
 def mock_import_module_with_success(monkeypatch):
     monkeypatch.setattr(importlib, "import_module", mock_import_module_success)
 
+@pytest.fixture
+def mock_import_module_with_success_multi_downversion(monkeypatch):
+    monkeypatch.setattr(importlib, "import_module", mock_import_module_success_multi_downversion)
+
 def test_search_loop(mock_import_module_with_loop):
     with pytest.raises(Exception):
         search('1.9.0', '2.0.0')
 
 def test_search_mission(mock_import_module_with_mission):
-    with pytest.raises(Exception):
+    with pytest.raises(MigratioNotFound):
         search('1.9.0', '2.0.0')
 
-def test_search_success():
+def test_search_success(mock_import_module_with_success):
     migration_path = search('1.9.0', '2.0.0')
     assert migration_path[0].revision == '1.10.0'
     assert migration_path[1].revision == '2.0.0'
+
+def test_search_success_multi_downversion(mock_import_module_with_success_multi_downversion):
+    migration_path = search('1.9.0', '2.0.0')
+    print(migration_path)
+    assert migration_path[0].revision == '1.10.2'
+    assert migration_path[1].revision == '2.0.0'

make/photon/prepare/utils/migration.py

@@ -4,7 +4,24 @@ import importlib
 import os
 from collections import deque
 
-from migrations import MIGRATION_BASE_DIR
+class MigratioNotFound(Exception): ...
+
+class MigrationVersion:
+    '''
+    The version used to migration
+
+    Arttribute:
+        name(str): version name like `1.0.0`
+        module: the python module object for a specific migration which contains migrate info, codes and templates
+        down_versions(list): previous versions that can migrated to this version
+    '''
+    def __init__(self, version: str):
+        self.name = version
+        self.module = importlib.import_module("migrations.version_{}".format(version.replace(".","_")))
+
+    @property
+    def down_versions(self):
+        return self.module.down_revisions
 
 def read_conf(path):
     with open(path) as f:
@@ -15,29 +32,35 @@ def read_conf(path):
             exit(-1)
     return d
 
-def _to_module_path(ver):
+def search(input_version: str, target_version: str) -> list :
-    return "migrations.version_{}".format(ver.replace(".","_"))
+    """
+    Find the migration path by BFS
+    Args:
+        input_version(str): The version migration start from
+        target_version(str): The target version migrated to
+    Returns:
+        list: the module of migrations in the upgrade path
+    """
+    upgrade_path = []
+    next_version, visited, q = {}, set(), deque()
+    q.append(target_version)
+    found = False
+    while q: # BFS to find a valid path
+        version = MigrationVersion(q.popleft())
+        visited.add(version.name)
+        if version.name == input_version:
+            found = True
+            break # break loop cause migration path found
+        for v in version.down_versions:
+            next_version[v] = version.name
+            if v not in (visited.union(q)):
+                q.append(v)
 
-def search(input_ver: str, target_ver: str) -> deque :
+    if not found:
-    """
+        raise MigratioNotFound('no migration path found to target version')
-    Search accept a input version and the target version.
+
-    Returns the module of migrations in the upgrade path
+    current_version = MigrationVersion(input_version)
-    """
+    while current_version.name != target_version:
-    upgrade_path, visited = deque(), set()
+        current_version = MigrationVersion(next_version[current_version.name])
-    while True:
+        upgrade_path.append(current_version)
-        module_path = _to_module_path(target_ver)
+    return list(map(lambda x: x.module, upgrade_path))
-        visited.add(target_ver)  # mark current version for loop finding
-        if os.path.isdir(os.path.join(MIGRATION_BASE_DIR, 'version_{}'.format(target_ver.replace(".","_")))):
-            module = importlib.import_module(module_path)
-            if module.revision == input_ver:    # migration path found
-                break
-            elif module.down_revision is None:  # migration path not found
-                raise Exception('no migration path found')
-            else:
-                upgrade_path.appendleft(module)
-                target_ver = module.down_revision
-                if target_ver in visited: # version visited before, loop found
-                    raise Exception('find a loop caused by {} on migration path'.format(target_ver))
-        else:
-            raise Exception('{} not dir'.format(os.path.join(MIGRATION_BASE_DIR, 'versions', target_ver.replace(".","_"))))
-    return upgrade_path