Merge pull request #4523 from ninjadq/migrator_1_5

Upgrade migrator to 1.5.0

make/photon/db/registry.sql

@@ -85,7 +85,7 @@ create table project_member (
  id int not null AUTO_INCREMENT,
  project_id int NOT NULL,
  entity_id int NOT NULL,
- entity_type char NOT NULL, ## u for user, g for user group
+ entity_type char(1) NOT NULL, ## u for user, g for user group
  role int NOT NULL,
  creation_time timestamp default CURRENT_TIMESTAMP,
  update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
@@ -218,7 +218,7 @@ create table img_scan_job (
  id int NOT NULL AUTO_INCREMENT,
  status varchar(64) NOT NULL,
  repository varchar(256) NOT NULL,
- tag   varchar(128) NOT NULL,
+ tag varchar(128) NOT NULL,
  digest varchar(128),
  #New job service only records uuid, for compatibility in this table both IDs are stored.
  job_uuid varchar(64),

tools/migration/db/db_meta.py

@@ -5,7 +5,7 @@ import sqlalchemy as sa
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker, relationship
 from sqlalchemy.dialects import mysql
-
+import datetime
 
 Base = declarative_base()
 
@@ -27,6 +27,17 @@ class User(Base):
     update_time = sa.Column(mysql.TIMESTAMP)
 
 
+class UserGroup(Base):
+    __tablename__ = 'user_group'
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    group_name = sa.Column(sa.String(128), nullable = False)
+    group_type = sa.Column(sa.Integer, server_default=sa.text("'0'"))
+    ldap_group_dn = sa.Column(sa.String(512), nullable=False)
+    creation_time = sa.Column(mysql.TIMESTAMP, server_default=sa.text("CURRENT_TIMESTAMP"))
+    update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
+
+
 class Properties(Base):
     __tablename__ = 'properties'
 
@@ -38,14 +49,15 @@ class Properties(Base):
 class ProjectMember(Base):
     __tablename__ = 'project_member'
 
-    project_id = sa.Column(sa.Integer(), primary_key = True)
+    id = sa.Column(sa.Integer, primary_key=True)
-    user_id = sa.Column(sa.Integer(), primary_key = True)
+    project_id = sa.Column(sa.Integer(), nullable=False)
+    entity_id = sa.Column(sa.Integer(), nullable=False)
+    entity_type = sa.Column(sa.String(1), nullable=False)
     role = sa.Column(sa.Integer(), nullable = False)
-    creation_time = sa.Column(mysql.TIMESTAMP, nullable = True)
+    creation_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP"))
-    update_time = sa.Column(mysql.TIMESTAMP, nullable = True)
+    update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
-    sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'], ),
+
-    sa.ForeignKeyConstraint(['role'], [u'role.role_id'], ),
+    __table_args__ = (sa.UniqueConstraint('project_id', 'entity_id', 'entity_type', name='unique_name_and_scope'),)
-    sa.ForeignKeyConstraint(['user_id'], [u'user.user_id'], ),
 
 
 class UserProjectRole(Base):
@@ -150,6 +162,7 @@ class ReplicationJob(Base):
     repository = sa.Column(sa.String(256), nullable=False)
     operation = sa.Column(sa.String(64), nullable=False)
     tags = sa.Column(sa.String(16384))
+    job_uuid = sa.Column(sa.String(64))
     creation_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP"))
     update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
 
@@ -207,6 +220,7 @@ class ImageScanJob(Base):
     repository = sa.Column(sa.String(256), nullable=False)
     tag = sa.Column(sa.String(128), nullable=False)
     digest = sa.Column(sa.String(128))
+    job_uuid = sa.Column(sa.String(64))
     creation_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP"))
     update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
 
@@ -230,3 +244,33 @@ class ClairVulnTimestamp(Base):
     id = sa.Column(sa.Integer, nullable=False, primary_key=True)
     namespace = sa.Column(sa.String(128), nullable=False, unique=True)
     last_update = sa.Column(mysql.TIMESTAMP)
+
+
+class HarborLabel(Base):
+    __tablename__ = "harbor_label"
+
+    id = sa.Column(sa.Integer, nullable=False, primary_key=True)
+    name = sa.Column(sa.String(128), nullable=False)
+    description = sa.Column(sa.Text)
+    color = sa.Column(sa.String(16))
+    level = sa.Column(sa.String(1), nullable=False)
+    scope = sa.Column(sa.String(1), nullable=False)
+    project_id = sa.Column(sa.Integer, nullable=False)
+    creation_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP"))
+    update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
+
+    __table_args__ = (sa.UniqueConstraint('name', 'scope', name='unique_name_and_scope'),)
+
+
+class HarborResourceLabel(Base):
+    __tablename__ = 'harbor_resource_label'
+ 
+    id = sa.Column(sa.Integer, nullable=False, primary_key=True)
+    label_id = sa.Column(sa.Integer, nullable=False)
+    resource_id =  sa.Column(sa.Integer)
+    resource_name = sa.Column(sa.String(256))
+    resource_type = sa.Column(sa.String(1), nullable=False)
+    creation_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP"))
+    update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
+
+    __table_args__ = (sa.UniqueConstraint('label_id', 'resource_id', 'resource_name', 'resource_type', name='unique_label_resource'),)

tools/migration/db/migration_harbor/versions/1_5_0.py

@@ -0,0 +1,90 @@
+# Copyright (c) 2008-2018 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""1.4.0 to 1.5.0
+
+Revision ID: 1.5.0
+Revises:
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '1.5.0'
+down_revision = '1.4.0'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+from db_meta import *
+
+from sqlalchemy.dialects import mysql
+
+Session = sessionmaker()
+
+def upgrade():
+    """
+    update schema&data
+    """
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+
+    # create table harbor_label
+    HarborLabel.__table__.create(bind)
+
+    # create table harbor_resource_label
+    HarborResourceLabel.__table__.create(bind)
+
+    # create user_group
+    UserGroup.__table__.create(bind)
+
+    # project member
+    op.drop_constraint('project_member_ibfk_1', 'project_member', type_='foreignkey')
+    op.drop_constraint('project_member_ibfk_2', 'project_member', type_='foreignkey')
+    op.drop_constraint('project_member_ibfk_3', 'project_member', type_='foreignkey')
+    op.drop_constraint('PRIMARY', 'project_member', type_='primary')
+    op.drop_index('user_id', 'project_member')
+    op.drop_index('role', 'project_member')
+    op.execute('ALTER TABLE project_member ADD id INT PRIMARY KEY AUTO_INCREMENT;')
+    op.alter_column('project_member', 'user_id', existing_type=sa.Integer, existing_nullable=False, new_column_name='entity_id')
+    op.alter_column('project_member', 'creation_time', existing_type=mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP"))
+    op.alter_column('project_member', 'update_time', existing_type=mysql.TIMESTAMP, server_default=sa.text("CURRENT_TIMESTAMP"), onupdate=sa.text("CURRENT_TIMESTAMP"))
+    op.add_column('project_member', sa.Column('entity_type', sa.String(1)))
+
+    session.query(ProjectMember).update({
+        ProjectMember.entity_type: 'u'
+    })
+    op.alter_column('project_member', 'entity_type', existing_type=sa.String(1), existing_nullable=True, nullable=False)
+
+    op.create_unique_constraint('unique_project_entity_type', 'project_member', ['project_id', 'entity_id', 'entity_type'])
+
+    # add job_uuid to replicationjob and img_scan_job
+    op.add_column('replication_job', sa.Column('job_uuid', sa.String(64)))
+    op.add_column('img_scan_job', sa.Column('job_uuid', sa.String(64)))
+
+    # add index to replication job
+    op.create_index('poid_status', 'replication_job', ['policy_id', 'status'])
+
+    # add index to img_scan_job
+    op.create_index('idx_status', 'img_scan_job', ['status'])
+    op.create_index('idx_digest', 'img_scan_job', ['digest'])
+    op.create_index('idx_uuid', 'img_scan_job', ['job_uuid'])
+    op.create_index('idx_repository_tag', 'img_scan_job', ['repository', 'tag'])
+
+    session.commit()
+
+def downgrade():
+    """
+    Downgrade has been disabled.
+    """