Fix: module path raise exception when it is loop

add test for loop

Signed-off-by: DQ <dengq@vmware.com>
This commit is contained in:
DQ 2020-03-23 15:32:15 +08:00
parent e8bb977ae1
commit 444678fe07
9 changed files with 194 additions and 81 deletions

View File

@ -10,6 +10,7 @@ click = "*"
[dev-packages]
pylint = "*"
pytest = "*"
[requires]
python_version = "3.7.5"

View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "fc86f400c4b9ebc8d4caf833b1e8d8c8962b7f4920b19632e9e4f4e8738ce7e2"
"sha256": "b165e0915808904ddcb9c552479da32f9f0611f09f462c442c2d1c9b347ced22"
},
"pipfile-spec": 6,
"requires": {
@ -18,19 +18,19 @@
"default": {
"click": {
"hashes": [
"sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
"sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
"sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc",
"sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a"
],
"index": "pypi",
"version": "==7.0"
"version": "==7.1.1"
},
"jinja2": {
"hashes": [
"sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f",
"sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de"
"sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250",
"sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49"
],
"index": "pypi",
"version": "==2.10.3"
"version": "==2.11.1"
},
"markupsafe": {
"hashes": [
@ -38,13 +38,16 @@
"sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161",
"sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235",
"sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5",
"sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42",
"sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff",
"sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b",
"sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1",
"sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e",
"sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183",
"sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66",
"sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b",
"sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1",
"sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15",
"sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1",
"sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e",
"sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b",
@ -61,7 +64,9 @@
"sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6",
"sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f",
"sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f",
"sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7"
"sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2",
"sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7",
"sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"
],
"version": "==1.1.1"
},
@ -81,6 +86,21 @@
],
"version": "==2.3.3"
},
"attrs": {
"hashes": [
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
],
"version": "==19.3.0"
},
"importlib-metadata": {
"hashes": [
"sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302",
"sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b"
],
"markers": "python_version < '3.8'",
"version": "==1.5.0"
},
"isort": {
"hashes": [
"sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1",
@ -121,6 +141,34 @@
],
"version": "==0.6.1"
},
"more-itertools": {
"hashes": [
"sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c",
"sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507"
],
"version": "==8.2.0"
},
"packaging": {
"hashes": [
"sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3",
"sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752"
],
"version": "==20.3"
},
"pluggy": {
"hashes": [
"sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
"sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
],
"version": "==0.13.1"
},
"py": {
"hashes": [
"sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa",
"sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0"
],
"version": "==1.8.1"
},
"pylint": {
"hashes": [
"sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd",
@ -129,44 +177,74 @@
"index": "pypi",
"version": "==2.4.4"
},
"pyparsing": {
"hashes": [
"sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f",
"sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"
],
"version": "==2.4.6"
},
"pytest": {
"hashes": [
"sha256:0e5b30f5cb04e887b91b1ee519fa3d89049595f428c1db76e73bd7f17b09b172",
"sha256:84dde37075b8805f3d1f392cc47e38a0e59518fb46a431cfdaf7cf1ce805f970"
],
"index": "pypi",
"version": "==5.4.1"
},
"six": {
"hashes": [
"sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
"sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
"sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a",
"sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"
],
"version": "==1.13.0"
"version": "==1.14.0"
},
"typed-ast": {
"hashes": [
"sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161",
"sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e",
"sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e",
"sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0",
"sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c",
"sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47",
"sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631",
"sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4",
"sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34",
"sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b",
"sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2",
"sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e",
"sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a",
"sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233",
"sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1",
"sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36",
"sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d",
"sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a",
"sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66",
"sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12"
"sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355",
"sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919",
"sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa",
"sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652",
"sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75",
"sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01",
"sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d",
"sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1",
"sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907",
"sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c",
"sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3",
"sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b",
"sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614",
"sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb",
"sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b",
"sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41",
"sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6",
"sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34",
"sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe",
"sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4",
"sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"
],
"markers": "implementation_name == 'cpython' and python_version < '3.8'",
"version": "==1.4.0"
"version": "==1.4.1"
},
"wcwidth": {
"hashes": [
"sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603",
"sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8"
],
"version": "==0.1.8"
},
"wrapt": {
"hashes": [
"sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"
],
"version": "==1.11.2"
},
"zipp": {
"hashes": [
"sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b",
"sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96"
],
"version": "==3.1.0"
}
}
}

View File

@ -3,7 +3,7 @@ import os, sys, importlib, shutil, glob
import click
from utils.misc import get_realpath
from migration.utils import read_conf, to_module_path, search
from migration.utils import read_conf, search
@click.command()

View File

@ -1,11 +1,13 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import os
import yaml
import click
import importlib
from string import Template
from collections import deque
BASE_DIR = os.path.dirname(__file__)
def read_conf(path):
with open(path) as f:
@ -16,36 +18,29 @@ def read_conf(path):
exit(-1)
return d
def to_module_path(ver):
def _to_module_path(ver):
return "migration.versions.{}".format(ver.replace(".","_"))
def search(input_ver: str, target_ver: str) -> list:
def search(input_ver: str, target_ver: str) -> deque :
"""
Search accept a input version and the target version.
Returns the module of migrations in the upgrade path
"""
def helper():
nonlocal basedir
nonlocal cur_target
while True:
module_path = to_module_path(cur_target)
if os.path.isdir(os.path.join(basedir, 'versions', cur_target.replace(".","_"))):
module = importlib.import_module(module_path)
yield module
if module.revision == input_ver:
return
elif module.down_revision is not None:
cur_target = module.down_revision
else:
return
upgrade_path, visited = deque(), set()
while True:
module_path = _to_module_path(target_ver)
visited.add(target_ver) # mark current version for loop finding
if os.path.isdir(os.path.join(BASE_DIR, 'versions', target_ver.replace(".","_"))):
module = importlib.import_module(module_path)
if module.revision == input_ver: # migration path found
break
elif module.down_revision is None: # migration path not found
raise Exception('no migration path found')
else:
click.echo('{} not dir'.format(os.path.join(basedir, 'versions', cur_target.replace(".","_"))))
return
basedir = os.path.dirname(__file__)
cur_target = target_ver
upgrade_path = list(helper())
upgrade_path.reverse()
if upgrade_path and upgrade_path[0].revision == input_ver and upgrade_path[-1].revision == target_ver:
return upgrade_path[1:]
else:
return []
upgrade_path.appendleft(module)
target_ver = module.down_revision
if target_ver in visited: # version visited before, loop found
raise Exception('find a loop caused by {} on migration path'.format(target_ver))
else:
raise Exception('{} not dir'.format(os.path.join(BASE_DIR, 'versions', target_ver.replace(".","_"))))
return upgrade_path

View File

@ -1,8 +1,5 @@
import os
import yaml
from jinja2 import Environment, FileSystemLoader, StrictUndefined
from migration import utils
revision = '1.10.0'

View File

@ -1,8 +1,5 @@
import os
import yaml
from jinja2 import Environment, FileSystemLoader, StrictUndefined
from migration import utils
revision = '1.9.0'

View File

@ -1,8 +1,5 @@
import os
import yaml
from jinja2 import Environment, FileSystemLoader, StrictUndefined
from migration import utils
revision = '2.0.0'

View File

@ -33,15 +33,6 @@ https:
# private_key: /your/private/key/path
{% endif %}
{% if internal_tls is defined %}
internal_tls:
# set enabled to true means internal tls is enabled
enabled: {{ internal_tls.enabled }}
# verify_client_cert used to decide whether verify client certificate
verify_client_cert: {{ internal_tls.verify_client_cert }}
# put your cert and key files on dir
dir: {{ internal_tls.dir }}
{% else %}
# # Uncomment following will enable tls communication between all harbor components
# internal_tls:
# # set enabled to true means internal tls is enabled
@ -50,7 +41,6 @@ internal_tls:
# verify_client_cert: false
# # put your cert and key files on dir
# dir: /etc/harbor/tls/internal
{% endif %}
# Uncomment external_url if you want to enable external proxy
# And when it enabled the hostname will no longer used
@ -206,7 +196,7 @@ log:
rotate_size: {{ log.local.rotate_size }}
# The directory on your host that store log
location: {{ log.local.location }}
{% if log.external_endpoint is defined%}
{% if log.external_endpoint is defined %}
external_endpoint:
# protocol used to transmit log to external endpoint, options is tcp or udp
protocol: {{ log.external_endpoint.protocol }}

View File

@ -0,0 +1,58 @@
import pytest
import importlib
from migration.utils import search
class mockModule:
def __init__(self, revision, down_revision):
self.revision = revision
self.down_revision = down_revision
def mock_import_module_loop(module_path: str):
loop_modules = {
'migration.versions.1_9_0': mockModule('1.9.0', None),
'migration.versions.1_10_0': mockModule('1.10.0', '2.0.0'),
'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0')
}
return loop_modules[module_path]
def mock_import_module_mission(module_path: str):
loop_modules = {
'migration.versions.1_9_0': mockModule('1.9.0', None),
'migration.versions.1_10_0': mockModule('1.10.0', None),
'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0')
}
return loop_modules[module_path]
def mock_import_module_success(module_path: str):
loop_modules = {
'migration.versions.1_9_0': mockModule('1.9.0', None),
'migration.versions.1_10_0': mockModule('1.10.0', '1.9.0'),
'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0')
}
return loop_modules[module_path]
@pytest.fixture
def mock_import_module_with_loop(monkeypatch):
monkeypatch.setattr(importlib, "import_module", mock_import_module_loop)
@pytest.fixture
def mock_import_module_with_mission(monkeypatch):
monkeypatch.setattr(importlib, "import_module", mock_import_module_mission)
@pytest.fixture
def mock_import_module_with_success(monkeypatch):
monkeypatch.setattr(importlib, "import_module", mock_import_module_success)
def test_search_loop(mock_import_module_with_loop):
with pytest.raises(Exception):
search('1.9.0', '2.0.0')
def test_search_mission(mock_import_module_with_mission):
with pytest.raises(Exception):
search('1.9.0', '2.0.0')
def test_search_success():
migration_path = search('1.9.0', '2.0.0')
assert migration_path[0].revision == '1.10.0'
assert migration_path[1].revision == '2.0.0'