Merge pull request #4005 from ninjadq/db_migrate_from_1_3_to_1_4

Update migration tool for v1.4

Makefile

@@ -104,7 +104,7 @@ NOTARYVERSION=v0.5.1
 MARIADBVERSION=$(VERSIONTAG)
 CLAIRVERSION=v2.0.1
 CLAIRDBVERSION=$(VERSIONTAG)
-MIGRATORVERSION=1.3
+MIGRATORVERSION=1.4
 
 #clarity parameters
 CLARITYIMAGE=vmware/harbor-clarity-ui-builder[:tag]

make/photon/db/registry.sql

@@ -69,7 +69,7 @@ create table project (
  owner_id int NOT NULL,
  # The max length of name controlled by API is 30, 
  # and 11 is reserved for marking the deleted project.
- name varchar (41) NOT NULL,
+ name varchar (255) NOT NULL,
  creation_time timestamp,
  update_time timestamp,
  deleted tinyint (1) DEFAULT 0 NOT NULL,
@@ -245,4 +245,4 @@ CREATE TABLE IF NOT EXISTS `alembic_version` (
     `version_num` varchar(32) NOT NULL
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
-insert into alembic_version values ('1.3.0');
+insert into alembic_version values ('1.4.0');

make/photon/db/registry_sqlite.sql

@@ -67,7 +67,7 @@ create table project (
  The max length of name controlled by API is 30, 
  and 11 is reserved for marking the deleted project.
 */
- name varchar (41) NOT NULL,
+ name varchar (255) NOT NULL,
  creation_time timestamp,
  update_time timestamp,
  deleted tinyint (1) DEFAULT 0 NOT NULL,
@@ -234,4 +234,4 @@ create table alembic_version (
     version_num varchar(32) NOT NULL
 );
 
-insert into alembic_version values ('1.3.0');
+insert into alembic_version values ('1.4.0');

src/common/dao/version.go

@@ -20,7 +20,7 @@ import (
 
 const (
 	// SchemaVersion is the version of database schema
-	SchemaVersion = "1.3.0"
+	SchemaVersion = "1.4.0"
 )
 
 // GetSchemaVersion return the version of database schema

tools/migration/changelog.md

@@ -57,10 +57,11 @@ Changelog for harbor database schema
   - delete column `public` from table `project`
   - add column `insecure` to table `replication_target`
 
-## 1.3.1
+## 1.4.0
 
   - add column `filters` to table `replication_policy`
   - add column `replicate_deletion` to table `replication_policy`
   - create table `replication_immediate_trigger`
   - add pk `id` to table `properties`
-  - remove pk index from colum 'k' of table `properties` 
+  - remove pk index from column 'k' of table `properties` 
+  - alter `name` length from 41 to 256 of table `project`

tools/migration/db_meta.py

@@ -30,7 +30,8 @@ class User(Base):
 class Properties(Base):
     __tablename__ = 'properties'
 
-    k = sa.Column(sa.String(64), primary_key = True)
+    id = sa.Column(sa.Integer, primary_key=True)
+    k = sa.Column(sa.String(64), unique=True)
     v = sa.Column(sa.String(128), nullable = False)
 
 
@@ -88,7 +89,7 @@ class Project(Base):
 
     project_id = sa.Column(sa.Integer, primary_key=True)
     owner_id = sa.Column(sa.ForeignKey(u'user.user_id'), nullable=False, index=True)
-    name = sa.Column(sa.String(30), nullable=False, unique=True)
+    name = sa.Column(sa.String(255), nullable=False, unique=True)
     creation_time = sa.Column(mysql.TIMESTAMP)
     update_time = sa.Column(mysql.TIMESTAMP)
     deleted = sa.Column(sa.Integer, nullable=False, server_default=sa.text("'0'"))
@@ -119,6 +120,8 @@ class ReplicationPolicy(Base):
     enabled = sa.Column(mysql.TINYINT(1), nullable=False, server_default=sa.text("'1'"))
     description = sa.Column(sa.Text)
     cron_str = sa.Column(sa.String(256))
+    filters = sa.Column(sa.String(1024))
+    replicate_deletion = sa.Column(mysql.TINYINT(1), nullable=False, server_default='0')
     start_time = sa.Column(mysql.TIMESTAMP)
     creation_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP"))
     update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
@@ -150,7 +153,19 @@ class ReplicationJob(Base):
     creation_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP"))
     update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
 
-    __table_args__ = (sa.Index('policy', "policy_id"),)
+    __table_args__ = (sa.Index('policy', 'policy_id'),)
+
+
+class ReplicationImmediateTrigger(Base):
+    __tablename__ = 'replication_immediate_trigger'
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    policy_id = sa.Column(sa.Integer, nullable=False)
+    namespace = sa.Column(sa.String(256), nullable=False)
+    on_push = sa.Column(mysql.TINYINT(1), nullable=False, server_default='0')
+    on_deletion = sa.Column(mysql.TINYINT(1), nullable=False, server_default='0')
+    creation_time = sa.Column(mysql.TIMESTAMP, server_default=sa.text("CURRENT_TIMESTAMP"))
+    update_time = sa.Column(mysql.TIMESTAMP, server_default=sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
 
 
 class Repository(Base):

tools/migration/migration_harbor/versions/1_4_0.py

@@ -0,0 +1,82 @@
+# Copyright (c) 2008-2018 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""1.3.0 to 1.4.0
+
+Revision ID: 1.3.0
+Revises:
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '1.4.0'
+down_revision = '1.3.0'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+from db_meta import *
+
+from sqlalchemy.dialects import mysql
+
+Session = sessionmaker()
+
+def upgrade():
+    """
+    update schema&data
+    """
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    # Alter column length of project name
+    op.alter_column('project', 'name', existing_type=sa.String(30), type_=sa.String(255), existing_nullable=False)
+
+    # Add columns in replication_policy table
+    op.add_column('replication_policy', sa.Column('filters', sa.String(1024)))
+    op.add_column('replication_policy', sa.Column('replicate_deletion', mysql.TINYINT(1), nullable=False, server_default='0'))
+
+    # create table replication_immediate_trigger
+    ReplicationImmediateTrigger.__table__.create(bind)
+
+    # Divided policies into unabled and enabled group
+    unenabled_policies = session.query(ReplicationPolicy).filter(ReplicationPolicy.enabled == 0)
+    enabled_policies = session.query(ReplicationPolicy).filter(ReplicationPolicy.enabled == 1)
+
+    # migrate enabeld policies
+    enabled_policies.update({
+        ReplicationPolicy.cron_str: '{"kind":"Immediate"}'
+    })
+    immediate_triggers = [ReplicationImmediateTrigger(
+        policy_id=policy.id,
+        namespace=session.query(Project).get(policy.project_id).name,
+        on_push=1,
+        on_deletion=1) for policy in enabled_policies]
+    session.add_all(immediate_triggers)
+
+    # migrate unenabled policies
+    unenabled_policies.update({
+        ReplicationPolicy.enabled: 1,
+        ReplicationPolicy.cron_str: '{"kind":"Manual"}'
+    })
+
+    op.drop_constraint('PRIMARY', 'properties', type_='primary')
+    op.create_unique_constraint('uq_properties_k', 'properties', ['k'])
+    op.execute('ALTER TABLE properties ADD id INT PRIMARY KEY AUTO_INCREMENT;')
+
+    session.commit()
+
+def downgrade():
+    """
+    Downgrade has been disabled.
+    """