Enhance: Refactor the migration structure

1. Refactor structure of migrate file
2. fix some previous bugs

Signed-off-by: DQ <dengq@vmware.com>

make/photon/prepare/commands/migrate.py

@@ -3,8 +3,7 @@ import os, sys, importlib, shutil, glob
 import click
 
 from utils.misc import get_realpath
-from migration.utils import read_conf, search
-
+from utils.migration import read_conf, search
 
 @click.command()
 @click.option('-i', '--input', 'input_', default='', help="The path of original config file")

make/photon/prepare/migrations/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+MIGRATION_BASE_DIR = os.path.dirname(__file__)

make/photon/prepare/migrations/version_1_10_0/__init__.py

@@ -1,12 +1,12 @@
 import os
 from jinja2 import Environment, FileSystemLoader, StrictUndefined
-from migration import utils
+from utils.migration import read_conf
 
 revision = '1.10.0'
 down_revision = '1.9.0'
 
 def migrate(input_cfg, output_cfg):
-    config_dict = utils.read_conf(input_cfg)
+    config_dict = read_conf(input_cfg)
 
     current_dir = os.path.dirname(__file__)
     tpl = Environment(

make/photon/prepare/migrations/version_1_10_0/harbor.yml.jinja

@@ -56,10 +56,10 @@ database:
   # The password for the root user of Harbor DB. Change this before any production use.
   password: {{ database.password}}
   # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
-  max_idle_conns: {{ database.max_idle_conns or 50}}
+  max_idle_conns: {{ database.max_idle_conns }}
   # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
   # Note: the default number of connections is 100 for postgres.
-  max_open_conns: {{ database.max_open_conns or 100}}
+  max_open_conns: {{ database.max_open_conns }}
 {% else %}
   # The password for the root user of Harbor DB. Change this before any production use.
   password: root123

make/photon/prepare/migrations/version_1_9_0/__init__.py

@@ -1,12 +1,12 @@
 import os
 from jinja2 import Environment, FileSystemLoader, StrictUndefined
-from migration import utils
+from utils.migration import read_conf
 
 revision = '1.9.0'
 down_revision = None
 
 def migrate(input_cfg, output_cfg):
-    config_dict = utils.read_conf(input_cfg)
+    config_dict = read_conf(input_cfg)
 
     this_dir = os.path.dirname(__file__)
     tpl = Environment(

make/photon/prepare/migrations/version_1_9_0/harbor.yml.jinja

@@ -160,8 +160,8 @@ external_database:
     username: {{ external_database.harbor.username }}
     password: {{ external_database.harbor.password }}
     ssl_mode: {{ external_database.harbor.ssl_mode }}
-    max_idle_conns: 2
-    max_open_conns: 0
+    max_idle_conns: 50
+    max_open_conns: 100
   clair:
     host: {{ external_database.clair.host }}
     port: {{ external_database.clair.port }}

make/photon/prepare/migrations/version_2_0_0/__init__.py

@@ -1,12 +1,12 @@
 import os
 from jinja2 import Environment, FileSystemLoader, StrictUndefined
-from migration import utils
+from utils.migration import read_conf
 
 revision = '2.0.0'
 down_revision = '1.10.0'
 
 def migrate(input_cfg, output_cfg):
-    config_dict = utils.read_conf(input_cfg)
+    config_dict = read_conf(input_cfg)
 
     current_dir = os.path.dirname(__file__)
     tpl = Environment(

make/photon/prepare/migrations/version_2_0_0/harbor.yml.jinja

@@ -65,10 +65,10 @@ database:
   # The password for the root user of Harbor DB. Change this before any production use.
   password: {{ database.password}}
   # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
-  max_idle_conns: {{ database.max_idle_conns or 50}}
+  max_idle_conns: {{ database.max_idle_conns }}
   # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
   # Note: the default number of connections is 100 for postgres.
-  max_open_conns: {{ database.max_open_conns or 100}}
+  max_open_conns: {{ database.max_open_conns }}
 {% else %}
   # The password for the root user of Harbor DB. Change this before any production use.
   password: root123
@@ -250,8 +250,8 @@ external_database:
     username: {{ external_database.harbor.username }}
     password: {{ external_database.harbor.password }}
     ssl_mode: {{ external_database.harbor.ssl_mode }}
-    max_idle_conns: {{ external_database.harbor.max_idle_conns }}
-    max_open_conns: {{ external_database.harbor.max_open_conns }}
+    max_idle_conns: {{ external_database.harbor.max_idle_conns}}
+    max_open_conns: {{ external_database.harbor.max_open_conns}}
   clair:
     host: {{ external_database.clair.host }}
     port: {{ external_database.clair.port }}
@@ -316,8 +316,8 @@ external_redis:
   jobservice_db_index: {{ external_redis.jobservice_db_index }}
   chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }}
   clair_db_index: {{ external_redis.clair_db_index }}
-  trivy_db_index: {{ external_redis.trivy_db_index }}
-  idle_timeout_seconds: {{ external_redis.idle_timeout_seconds }}
+  trivy_db_index: 5
+  idle_timeout_seconds: 30
 {% else %}
 # Umcomments external_redis if using external Redis server
 # external_redis:

make/photon/prepare/tests/migrations/utils_test.py

@@ -1,7 +1,7 @@
 import pytest
 import importlib
 
-from migration.utils import search
+from utils.migration import search
 
 class mockModule:
     def __init__(self, revision, down_revision):

make/photon/prepare/utils/migration.py

@@ -1,13 +1,11 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-import os
 import yaml
 import click
 import importlib
+import os
 
 from collections import deque
 
-BASE_DIR = os.path.dirname(__file__)
+from migrations import MIGRATION_BASE_DIR
 
 def read_conf(path):
     with open(path) as f:
@@ -19,7 +17,7 @@ def read_conf(path):
     return d
 
 def _to_module_path(ver):
-    return "migration.versions.{}".format(ver.replace(".","_"))
+    return "migrations.version_{}".format(ver.replace(".","_"))
 
 def search(input_ver: str, target_ver: str) -> deque :
     """
@@ -30,7 +28,7 @@ def search(input_ver: str, target_ver: str) -> deque :
     while True:
         module_path = _to_module_path(target_ver)
         visited.add(target_ver)  # mark current version for loop finding
-        if os.path.isdir(os.path.join(BASE_DIR, 'versions', target_ver.replace(".","_"))):
+        if os.path.isdir(os.path.join(MIGRATION_BASE_DIR, 'version_{}'.format(target_ver.replace(".","_")))):
             module = importlib.import_module(module_path)
             if module.revision == input_ver:    # migration path found
                 break
@@ -42,5 +40,5 @@ def search(input_ver: str, target_ver: str) -> deque :
                 if target_ver in visited: # version visited before, loop found
                     raise Exception('find a loop caused by {} on migration path'.format(target_ver))
         else:
-            raise Exception('{} not dir'.format(os.path.join(BASE_DIR, 'versions', target_ver.replace(".","_"))))
+            raise Exception('{} not dir'.format(os.path.join(MIGRATION_BASE_DIR, 'versions', target_ver.replace(".","_"))))
     return upgrade_path