Enhance: Support multi downversion in migration

1. Change down version to list to accept multi verstion value
2. Update search function use BFS to find migration path
2. Add test case

Signed-off-by: DQ <dengq@vmware.com>
This commit is contained in:
DQ 2020-06-28 17:49:14 +08:00
parent 8bcffb0a28
commit 4617e0ff38
7 changed files with 98 additions and 52 deletions

View File

@ -14,6 +14,9 @@ from migrations import accept_versions
def migrate(input_, output, target):
"""
migrate command will migrate config file style to specific version
:input_: is the path of the original config file
:output: is the destination path of config file, the generated configs will storage in it
:target: is the the target version of config file will upgrade to
"""
if target not in accept_versions:
click.echo('target version {} not supported'.format(target))

View File

@ -3,7 +3,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined
from utils.migration import read_conf
revision = '1.10.0'
down_revision = '1.9.0'
down_revisions = ['1.9.0']
def migrate(input_cfg, output_cfg):
config_dict = read_conf(input_cfg)

View File

@ -3,14 +3,14 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined
from utils.migration import read_conf
revision = '1.9.0'
down_revision = None
down_revisions = []
def migrate(input_cfg, output_cfg):
config_dict = read_conf(input_cfg)
this_dir = os.path.dirname(__file__)
current_dir = os.path.dirname(__file__)
tpl = Environment(
loader=FileSystemLoader(this_dir),
loader=FileSystemLoader(current_dir),
undefined=StrictUndefined,
trim_blocks=True,
lstrip_blocks=True

View File

@ -48,7 +48,7 @@ harbor_admin_password: {{ harbor_admin_password }}
# Harbor DB configuration
database:
# The password for the root user of Harbor DB. Change this before any production use.
password: {{ database.password}}
password: {{ database.password }}
# The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
max_idle_conns: 50
# The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.

View File

@ -3,7 +3,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined
from utils.migration import read_conf
revision = '2.0.0'
down_revision = '1.10.0'
down_revisions = ['1.10.0']
def migrate(input_cfg, output_cfg):
config_dict = read_conf(input_cfg)

View File

@ -1,36 +1,46 @@
import pytest
import importlib
from utils.migration import search
from utils.migration import search, MigratioNotFound
class mockModule:
def __init__(self, revision, down_revision):
def __init__(self, revision: str, down_revisions: list):
self.revision = revision
self.down_revision = down_revision
self.down_revisions = down_revisions
def mock_import_module_loop(module_path: str):
loop_modules = {
'migration.versions.1_9_0': mockModule('1.9.0', None),
'migration.versions.1_10_0': mockModule('1.10.0', '2.0.0'),
'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0')
modules = {
'migrations.version_1_9_0': mockModule('1.9.0', []),
'migrations.version_1_10_0': mockModule('1.10.0', ['2.0.0']),
'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0'])
}
return loop_modules[module_path]
return modules[module_path]
def mock_import_module_mission(module_path: str):
loop_modules = {
'migration.versions.1_9_0': mockModule('1.9.0', None),
'migration.versions.1_10_0': mockModule('1.10.0', None),
'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0')
modules = {
'migrations.version_1_9_0': mockModule('1.9.0', []),
'migrations.version_1_10_0': mockModule('1.10.0', []),
'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0'])
}
return loop_modules[module_path]
return modules[module_path]
def mock_import_module_success(module_path: str):
loop_modules = {
'migration.versions.1_9_0': mockModule('1.9.0', None),
'migration.versions.1_10_0': mockModule('1.10.0', '1.9.0'),
'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0')
modules = {
'migrations.version_1_9_0': mockModule('1.9.0', []),
'migrations.version_1_10_0': mockModule('1.10.0', ['1.9.0']),
'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0'])
}
return loop_modules[module_path]
return modules[module_path]
def mock_import_module_success_multi_downversion(module_path: str):
modules = {
'migrations.version_1_9_0': mockModule('1.9.0', []),
'migrations.version_1_10_0': mockModule('1.10.0', ['1.9.0']),
'migrations.version_1_10_1': mockModule('1.10.1', ['1.9.0']),
'migrations.version_1_10_2': mockModule('1.10.2', ['1.9.0']),
'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0', '1.10.1', '1.10.2'])
}
return modules[module_path]
@pytest.fixture
def mock_import_module_with_loop(monkeypatch):
@ -44,15 +54,25 @@ def mock_import_module_with_mission(monkeypatch):
def mock_import_module_with_success(monkeypatch):
monkeypatch.setattr(importlib, "import_module", mock_import_module_success)
@pytest.fixture
def mock_import_module_with_success_multi_downversion(monkeypatch):
monkeypatch.setattr(importlib, "import_module", mock_import_module_success_multi_downversion)
def test_search_loop(mock_import_module_with_loop):
with pytest.raises(Exception):
search('1.9.0', '2.0.0')
def test_search_mission(mock_import_module_with_mission):
with pytest.raises(Exception):
with pytest.raises(MigratioNotFound):
search('1.9.0', '2.0.0')
def test_search_success():
def test_search_success(mock_import_module_with_success):
migration_path = search('1.9.0', '2.0.0')
assert migration_path[0].revision == '1.10.0'
assert migration_path[1].revision == '2.0.0'
def test_search_success_multi_downversion(mock_import_module_with_success_multi_downversion):
migration_path = search('1.9.0', '2.0.0')
print(migration_path)
assert migration_path[0].revision == '1.10.2'
assert migration_path[1].revision == '2.0.0'

View File

@ -4,7 +4,24 @@ import importlib
import os
from collections import deque
from migrations import MIGRATION_BASE_DIR
class MigratioNotFound(Exception): ...
class MigrationVersion:
'''
The version used to migration
Arttribute:
name(str): version name like `1.0.0`
module: the python module object for a specific migration which contains migrate info, codes and templates
down_versions(list): previous versions that can migrated to this version
'''
def __init__(self, version: str):
self.name = version
self.module = importlib.import_module("migrations.version_{}".format(version.replace(".","_")))
@property
def down_versions(self):
return self.module.down_revisions
def read_conf(path):
with open(path) as f:
@ -15,29 +32,35 @@ def read_conf(path):
exit(-1)
return d
def _to_module_path(ver):
return "migrations.version_{}".format(ver.replace(".","_"))
def search(input_version: str, target_version: str) -> list :
"""
Find the migration path by BFS
Args:
input_version(str): The version migration start from
target_version(str): The target version migrated to
Returns:
list: the module of migrations in the upgrade path
"""
upgrade_path = []
next_version, visited, q = {}, set(), deque()
q.append(target_version)
found = False
while q: # BFS to find a valid path
version = MigrationVersion(q.popleft())
visited.add(version.name)
if version.name == input_version:
found = True
break # break loop cause migration path found
for v in version.down_versions:
next_version[v] = version.name
if v not in (visited.union(q)):
q.append(v)
def search(input_ver: str, target_ver: str) -> deque :
"""
Search accept a input version and the target version.
Returns the module of migrations in the upgrade path
"""
upgrade_path, visited = deque(), set()
while True:
module_path = _to_module_path(target_ver)
visited.add(target_ver) # mark current version for loop finding
if os.path.isdir(os.path.join(MIGRATION_BASE_DIR, 'version_{}'.format(target_ver.replace(".","_")))):
module = importlib.import_module(module_path)
if module.revision == input_ver: # migration path found
break
elif module.down_revision is None: # migration path not found
raise Exception('no migration path found')
else:
upgrade_path.appendleft(module)
target_ver = module.down_revision
if target_ver in visited: # version visited before, loop found
raise Exception('find a loop caused by {} on migration path'.format(target_ver))
else:
raise Exception('{} not dir'.format(os.path.join(MIGRATION_BASE_DIR, 'versions', target_ver.replace(".","_"))))
return upgrade_path
if not found:
raise MigratioNotFound('no migration path found to target version')
current_version = MigrationVersion(input_version)
while current_version.name != target_version:
current_version = MigrationVersion(next_version[current_version.name])
upgrade_path.append(current_version)
return list(map(lambda x: x.module, upgrade_path))