From b2af17f5f6b1234c3288cacd8eef263c05015303 Mon Sep 17 00:00:00 2001 From: saga92 Date: Mon, 9 May 2016 14:11:34 +0800 Subject: [PATCH 1/5] add data migration module --- Deploy/db/registry.sql | 6 + migration/Dockerfile | 23 ++++ migration/README.md | 51 +++++++ migration/alembic.sql | 4 + migration/alembic.tpl | 68 ++++++++++ migration/install.sh | 3 + migration/migration.cfg | 4 + migration/migration_harbor/README | 1 + migration/migration_harbor/env.py | 85 ++++++++++++ migration/migration_harbor/env.pyc | Bin 0 -> 1814 bytes migration/migration_harbor/script.py.mako | 24 ++++ migration/migration_harbor/versions/0_1_1.py | 128 ++++++++++++++++++ migration/migration_harbor/versions/0_1_1.pyc | Bin 0 -> 5022 bytes migration/run.sh | 94 +++++++++++++ 14 files changed, 491 insertions(+) create mode 100644 migration/Dockerfile create mode 100644 migration/README.md create mode 100644 migration/alembic.sql create mode 100644 migration/alembic.tpl create mode 100755 migration/install.sh create mode 100644 migration/migration.cfg create mode 100644 migration/migration_harbor/README create mode 100644 migration/migration_harbor/env.py create mode 100644 migration/migration_harbor/env.pyc create mode 100644 migration/migration_harbor/script.py.mako create mode 100644 migration/migration_harbor/versions/0_1_1.py create mode 100644 migration/migration_harbor/versions/0_1_1.pyc create mode 100755 migration/run.sh diff --git a/Deploy/db/registry.sql b/Deploy/db/registry.sql index bd0644b33..1fd0eafa3 100644 --- a/Deploy/db/registry.sql +++ b/Deploy/db/registry.sql @@ -111,3 +111,9 @@ create table properties ( insert into properties (k, v) values ('schema_version', '0.1.1'); + +CREATE TABLE IF NOT EXISTS `alembic_version` ( + `version_num` varchar(32) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +insert into alembic_version values ('0.1.1'); diff --git a/migration/Dockerfile b/migration/Dockerfile new file mode 100644 index 000000000..eb961aac9 --- /dev/null +++ b/migration/Dockerfile @@ -0,0 +1,23 @@ +FROM mysql:5.6 + +MAINTAINER bhe@vmware.com + +RUN sed -i -e 's/us.archive.ubuntu.com/archive.ubuntu.com/g' /etc/apt/sources.list + +RUN apt-get update + +RUN apt-get install -y curl python python-pip git python-mysqldb + +RUN pip install alembic + +RUN mkdir -p /harbor-migration + +WORKDIR /harbor-migration + +COPY ./ ./ + +COPY ./migration.cfg ./ + +RUN ./install.sh + +ENTRYPOINT ["./run.sh"] diff --git a/migration/README.md b/migration/README.md new file mode 100644 index 000000000..1d013e47f --- /dev/null +++ b/migration/README.md @@ -0,0 +1,51 @@ +# migration +Migration is a module for migrating database schema between different version of project [harbor](https://github.com/vmware/harbor) + +**WARNING!!** You must backup your data before migrating + +###installation +- step 1: modify migration.cfg +- step 2: build image from dockerfile + ``` + cd harbor-migration + + docker build -t your-image-name . + ``` + +###migration operation +- show instruction of harbor-migration + + ```docker run your-image-name help``` + +- create backup file in `/path/to/backup` + + ``` + docker run -ti -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup your-image-name backup + ``` + +- restore from backup file in `/path/to/backup` + + ``` + docker run -ti -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup your-image-name restore + ``` + +- perform database schema upgrade + + ```docker run -ti -v /data/database:/var/lib/mysql your-image-name up head``` + +- perform database schema downgrade(downgrade has been disabled) + + ```docker run -v /data/database:/var/lib/mysql your-image-name down base``` + +###migration step +- step 1: stop and remove harbor service + + ``` + docker-compose stop && docker-compose rm -f + ``` +- step 2: perform migration operation +- step 3: rebuild newest harbor images and restart service + + ``` + docker-compose build && docker-compose up -d + ``` diff --git a/migration/alembic.sql b/migration/alembic.sql new file mode 100644 index 000000000..21dc7a1de --- /dev/null +++ b/migration/alembic.sql @@ -0,0 +1,4 @@ +use `registry`; +CREATE TABLE IF NOT EXISTS `alembic_version` ( + `version_num` varchar(32) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/migration/alembic.tpl b/migration/alembic.tpl new file mode 100644 index 000000000..548c6028d --- /dev/null +++ b/migration/alembic.tpl @@ -0,0 +1,68 @@ +echo " +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = migration_harbor + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to migration_harbor/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat migration_harbor/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = mysql://$db_username:$db_password@localhost:$db_port/$db_name + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S" diff --git a/migration/install.sh b/migration/install.sh new file mode 100755 index 000000000..4d707407c --- /dev/null +++ b/migration/install.sh @@ -0,0 +1,3 @@ +#!/bin/bash +source ./migration.cfg +source ./alembic.tpl > ./alembic.ini diff --git a/migration/migration.cfg b/migration/migration.cfg new file mode 100644 index 000000000..a383853ac --- /dev/null +++ b/migration/migration.cfg @@ -0,0 +1,4 @@ +db_username="root" +db_password="root123" +db_port="3306" +db_name="registry" diff --git a/migration/migration_harbor/README b/migration/migration_harbor/README new file mode 100644 index 000000000..98e4f9c44 --- /dev/null +++ b/migration/migration_harbor/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/migration/migration_harbor/env.py b/migration/migration_harbor/env.py new file mode 100644 index 000000000..646f39862 --- /dev/null +++ b/migration/migration_harbor/env.py @@ -0,0 +1,85 @@ +# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata + +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migration/migration_harbor/env.pyc b/migration/migration_harbor/env.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7899987c8414b352be74b3e9cd0086cceda1f730 GIT binary patch literal 1814 zcma)+TaOzx6o8LslF6mX?o!$!#EaiHNGN?rsDdsaRjQ&gEn1Y9ku!6WIQ4kav1c!$ zl`54Nc;SgZ$Y0y53mzl8Oq}gEhE^)T$tBXo*2kr(F5P7fMdaX35WjkF@(h+ z$#2uA;k>eiv=vFL{3;~>38_@gW_yrGvZ(SL_g<|G2CXgDD3z0!Xv|#7$Jf%m)cd`* zo8YjuB=Z_Wv_a((Wl`BvFYyAcw1@>o@#W($o;{Jik}EQIp|oKST_$JB7>d8T34e}P zm^U70AIsn#OYMWVjb@9^*TH?ovf$C>wcdMKHGbdt&ZuJ?+$*C@zQyu--qgm&B;wy= zzB^TK&}SujRVV^xN;0E8*2-inZ40*ue0jc>4iHugI!fED+Vl0KfxOY}8M=v-H3Cd8 z>IVILv|1s-miby)r*c*ctk#Xq4)+|WOZ#+sM*r>LKA_p{k!-8#RaHMkdokZ%`v$w#!~m|vJ2i;;Lgnu?rP3_ph&_;oZPe!YNQ6vmh-CYFic7Tz6^DM}no)&>i5kxYGfi5a8$3)r5IMm~#WUA8GhE%!`cT2PFd&&J`Ze@04;!AIP_jyvf@oQMbFq&MnK O2gz`BZ*+em#?c=_|Fw4j literal 0 HcmV?d00001 diff --git a/migration/migration_harbor/script.py.mako b/migration/migration_harbor/script.py.mako new file mode 100644 index 000000000..43c09401b --- /dev/null +++ b/migration/migration_harbor/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migration/migration_harbor/versions/0_1_1.py b/migration/migration_harbor/versions/0_1_1.py new file mode 100644 index 000000000..0f21b5436 --- /dev/null +++ b/migration/migration_harbor/versions/0_1_1.py @@ -0,0 +1,128 @@ +# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""0.1.0 to 0.1.1 + +Revision ID: 0.1.1 +Revises: +Create Date: 2016-04-18 18:32:14.101897 + +""" + +# revision identifiers, used by Alembic. +revision = '0.1.1' +down_revision = None +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker, relationship +from datetime import datetime + +Session = sessionmaker() + +Base = declarative_base() + +class Properties(Base): + __tablename__ = 'properties' + + k = sa.Column(sa.String(64), primary_key = True) + v = sa.Column(sa.String(128), nullable = False) + +class ProjectMember(Base): + __tablename__ = 'project_member' + + project_id = sa.Column(sa.Integer(), primary_key = True) + user_id = sa.Column(sa.Integer(), primary_key = True) + role = sa.Column(sa.Integer(), nullable = False) + creation_time = sa.Column(sa.DateTime(), nullable = True) + update_time = sa.Column(sa.DateTime(), nullable = True) + sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'], ), + sa.ForeignKeyConstraint(['role'], [u'role.role_id'], ), + sa.ForeignKeyConstraint(['user_id'], [u'user.user_id'], ), + +class UserProjectRole(Base): + __tablename__ = 'user_project_role' + + upr_id = sa.Column(sa.Integer(), primary_key = True) + user_id = sa.Column(sa.Integer(), sa.ForeignKey('user.user_id')) + pr_id = sa.Column(sa.Integer(), sa.ForeignKey('project_role.pr_id')) + project_role = relationship("ProjectRole") + +class ProjectRole(Base): + __tablename__ = 'project_role' + + pr_id = sa.Column(sa.Integer(), primary_key = True) + project_id = sa.Column(sa.Integer(), nullable = False) + role_id = sa.Column(sa.Integer(), nullable = False) + sa.ForeignKeyConstraint(['role_id'], [u'role.role_id']) + sa.ForeignKeyConstraint(['project_id'], [u'project.project_id']) + +class Access(Base): + __tablename__ = 'access' + + access_id = sa.Column(sa.Integer(), primary_key = True) + access_code = sa.Column(sa.String(1)) + comment = sa.Column(sa.String(30)) + +def upgrade(): + """ + update schema&data + """ + bind = op.get_bind() + session = Session(bind=bind) + + #delete M from table access + acc = session.query(Access).filter_by(access_id=1).first() + session.delete(acc) + + #create table property + Properties.__table__.create(bind) + session.add(Properties(k='schema_version', v='0.1.1')) + + #create table project_member + ProjectMember.__table__.create(bind) + + #fill data + join_result = session.query(UserProjectRole).join(UserProjectRole.project_role).all() + for result in join_result: + session.add(ProjectMember(project_id=result.project_role.project_id, \ + user_id=result.user_id, role=result.project_role.role_id, \ + creation_time=datetime.now(), update_time=datetime.now())) + + #drop user_project_role table before drop project_role + #because foreign key constraint + op.drop_table('user_project_role') + op.drop_table('project_role') + + #add column to table project + op.add_column('project', sa.Column('update_time', sa.DateTime(), nullable=True)) + + #add column to table role + op.add_column('role', sa.Column('role_mask', sa.Integer(), server_default=sa.text(u"'0'"), nullable=False)) + + #add column to table user + op.add_column('user', sa.Column('creation_time', sa.DateTime(), nullable=True)) + op.add_column('user', sa.Column('sysadmin_flag', sa.Integer(), nullable=True)) + op.add_column('user', sa.Column('update_time', sa.DateTime(), nullable=True)) + session.commit() + +def downgrade(): + """ + Downgrade has been disabled. + """ + pass diff --git a/migration/migration_harbor/versions/0_1_1.pyc b/migration/migration_harbor/versions/0_1_1.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d1c7dfee49f43cf97c4f7e70f9f85b84463cbd0 GIT binary patch literal 5022 zcmcgwZEqXL5#HlT6iJa1^nsFANzm$rN5v*pnaaXBae!r8Vz8RxZ&>1?C#vo%(Jt!|J`bYf9gIR zsp4bf_dcGqV=3h+H3YTP&{E!#8rnjagf1(uqC8s#RTZwN;fk`@Us6F$4eKIXRzX9B zt7^EK%c?49s<5Sot-QaYf;APctKqr|+iKX(xiuARsBl9KHdM zw~el6MuD5Sng7B#M{a6*9FR3ZQj^lA(0yhSVI(GiUL2*T{v@BE#tC3o=7%P0p!54A zoCMkTcl8~iL|ctlOEqi z)||}O^zW0m3NlIj&`r*qXXecR3z@Vung)S;6qp_@6^`RXZfG1w=5d@b_ND>5EhylJ z%7p@VC3UkZeh%Yc8b;E5k|ln0lJOP4Or|E|jz7z2*=KH$=KU8vbX7?t0QQnN&h}vW zc+ZPR*x=pJKcN@mX!q1jj^bo5|0(%AIny-nI-Zn`s#UGr zvgBRhma!RSzct~JNiKTkpU}R{GsqM;=Ud%!PEA;V%O^u6h3 z+3lBXadr|1X3894qfg@aUbX_pG&Kp@B4oQ?Fna3{YObIepPGX;!%Wj88Tu&7%!x@f zvr>0Ru946TeHBC^mMHKgvyEd%LiB6XEe4gFzr4l)}sK-@Ts0ydGt|H3P0N{xYBO1(xKMY5${v2?XwqhYDBt#B9ts>ZXD8lo}k zC+Js1VMV6TnDIIfNiVLYs~c4rEx>_+S0zsLLJk381?~% zRThSA&W6FD|26$wZ);)tZ!u%8cbU1|{aZB7@)T|b+i>?|DJzcr$kR-JiFeI&M=F+d zbZSD^De67F#rR;b$n9aFRA7~$Z;;#sk@zl-gS^0DP6(uhZu%^r*(8|N@yxM14YH{X zg^%_>5@UF*NtKbFrLGtHk#iilC%x}+dOUeKa$+)vR?AvX@(Y8UO3$YzIn#`F$;4wn z$gs}QnFx>lB+Ue3JQJAANZl%WMbV8PB7oF4CZkoZ=V@+~CJ2aqDXVF%+qbQj-L`JpZMLjy))lK;?b)B$ zJ&WHF^?0`-7#-qC{{g~$dK>QGwDsg#zknY$6PJSlDZmy~|3z}<{l+)?xC@kWAO;|& z?4$56#{-`Pb7XRK06hhY5w?mvV!AvQ7cJz_kNG-0Js#+vfTncvgCOYU=Ro&3PP*Bt z=_1B66Y70PC3p0IElDLwsy@5BB)r3GhCq*7CYhk`u=O#?0f|IC&nXW#(<0&|0HrUt zN_`(ugLkWP(tezTnqVT;vL-%= z*W^V$#eW}yLnX{e@haZbzoG6UkkU!gdY7K!UjH)xOXAmpIgG2Sjgo=iN*n1-cXh0G TrH*f>+G%!boqA`b{^5TCFUSJ1 literal 0 HcmV?d00001 diff --git a/migration/run.sh b/migration/run.sh new file mode 100755 index 000000000..806378ff5 --- /dev/null +++ b/migration/run.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +source ./migration.cfg + +WAITTIME=60 + +DBCNF="-hlocalhost -u${db_username}" + +#prevent shell to print insecure message +export MYSQL_PWD="${db_password}" + +if [[ $1 = "help" || $1 = "h" || $# = 0 ]]; then + echo "Usage:" + echo "backup perform database backup" + echo "restore perform database restore" + echo "up, upgrade perform database schema upgrade" + echo "h, help usage help" + exit 0 +fi + +if [[ $1 = "up" || $1 = "upgrade" ]]; then + echo "Please backup before upgrade." + read -p "Enter y to continue updating or n to abort:" ans + case $ans in + [Yy]* ) + ;; + [Nn]* ) + exit 0 + ;; + * ) echo "illegal answer: $ans. Upgrade abort!!" + exit 1 + ;; + esac + +fi + +echo 'Trying to start mysql server...' +DBRUN=0 +nohup mysqld 2>&1 > ./nohup.log& +for i in $(seq 1 $WAITTIME); do + echo "$(/usr/sbin/service mysql status)" + if [[ "$(/usr/sbin/service mysql status)" =~ "not running" ]]; then + sleep 1 + else + DBRUN=1 + break + fi +done + +if [[ $DBRUN -eq 0 ]]; then + echo "timeout. Can't run mysql server." + exit 1 +fi + +key="$1" +case $key in +up|upgrade) + VERSION="$2" + if [[ -z $VERSION ]]; then + VERSION="head" + echo "Version is not specified. Default version is head." + fi + echo "Performing upgrade ${VERSION}..." + if [[ $(mysql $DBCNF -N -s -e "select count(*) from information_schema.tables \ + where table_schema='registry' and table_name='alembic_version';") -eq 0 ]]; then + echo "table alembic_version does not exist. Trying to initial alembic_version." + mysql $DBCNF < ./alembic.sql + #compatible with version 0.1.0 and 0.1.1 + if [[ $(mysql $DBCNF -N -s -e "select count(*) from information_schema.tables \ + where table_schema='registry' and table_name='properties'") -eq 0 ]]; then + echo "table properties does not exist. The version of registry is 0.1.0" + else + echo "The version of registry is 0.1.1" + mysql $DBCNF -e "insert into registry.alembic_version values ('0.1.1')" + fi + fi + alembic -c ./alembic.ini upgrade ${VERSION} + echo "Upgrade performed." + ;; +backup) + echo "Performing backup..." + mysqldump $DBCNF --add-drop-database --databases registry > ./backup/registry.sql + echo "Backup performed." + ;; +restore) + echo "Performing restore..." + mysql $DBCNF < ./backup/registry.sql + echo "Restore performed." + ;; +*) + echo "unknown option" + exit 0 + ;; +esac From 65c0c473684c7fbf5510a8d3a14b46fea0bc0ec3 Mon Sep 17 00:00:00 2001 From: saga92 Date: Mon, 9 May 2016 14:14:27 +0800 Subject: [PATCH 2/5] Delete README --- migration/migration_harbor/README | 1 - 1 file changed, 1 deletion(-) delete mode 100644 migration/migration_harbor/README diff --git a/migration/migration_harbor/README b/migration/migration_harbor/README deleted file mode 100644 index 98e4f9c44..000000000 --- a/migration/migration_harbor/README +++ /dev/null @@ -1 +0,0 @@ -Generic single-database configuration. \ No newline at end of file From 7617f4c9a3084fd0a40a363551433e681b45b2a4 Mon Sep 17 00:00:00 2001 From: saga92 Date: Mon, 9 May 2016 14:18:20 +0800 Subject: [PATCH 3/5] delete pyc file --- .gitignore | 1 + migration/migration_harbor/env.pyc | Bin 1814 -> 0 bytes migration/migration_harbor/versions/0_1_1.pyc | Bin 5022 -> 0 bytes 3 files changed, 1 insertion(+) delete mode 100644 migration/migration_harbor/env.pyc delete mode 100644 migration/migration_harbor/versions/0_1_1.pyc diff --git a/.gitignore b/.gitignore index 6b04c4ad3..b53274b48 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ Deploy/config/ui/app.conf Deploy/config/db/env Deploy/harbor.cfg ui/ui +*.pyc diff --git a/migration/migration_harbor/env.pyc b/migration/migration_harbor/env.pyc deleted file mode 100644 index 7899987c8414b352be74b3e9cd0086cceda1f730..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1814 zcma)+TaOzx6o8LslF6mX?o!$!#EaiHNGN?rsDdsaRjQ&gEn1Y9ku!6WIQ4kav1c!$ zl`54Nc;SgZ$Y0y53mzl8Oq}gEhE^)T$tBXo*2kr(F5P7fMdaX35WjkF@(h+ z$#2uA;k>eiv=vFL{3;~>38_@gW_yrGvZ(SL_g<|G2CXgDD3z0!Xv|#7$Jf%m)cd`* zo8YjuB=Z_Wv_a((Wl`BvFYyAcw1@>o@#W($o;{Jik}EQIp|oKST_$JB7>d8T34e}P zm^U70AIsn#OYMWVjb@9^*TH?ovf$C>wcdMKHGbdt&ZuJ?+$*C@zQyu--qgm&B;wy= zzB^TK&}SujRVV^xN;0E8*2-inZ40*ue0jc>4iHugI!fED+Vl0KfxOY}8M=v-H3Cd8 z>IVILv|1s-miby)r*c*ctk#Xq4)+|WOZ#+sM*r>LKA_p{k!-8#RaHMkdokZ%`v$w#!~m|vJ2i;;Lgnu?rP3_ph&_;oZPe!YNQ6vmh-CYFic7Tz6^DM}no)&>i5kxYGfi5a8$3)r5IMm~#WUA8GhE%!`cT2PFd&&J`Ze@04;!AIP_jyvf@oQMbFq&MnK O2gz`BZ*+em#?c=_|Fw4j diff --git a/migration/migration_harbor/versions/0_1_1.pyc b/migration/migration_harbor/versions/0_1_1.pyc deleted file mode 100644 index 4d1c7dfee49f43cf97c4f7e70f9f85b84463cbd0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5022 zcmcgwZEqXL5#HlT6iJa1^nsFANzm$rN5v*pnaaXBae!r8Vz8RxZ&>1?C#vo%(Jt!|J`bYf9gIR zsp4bf_dcGqV=3h+H3YTP&{E!#8rnjagf1(uqC8s#RTZwN;fk`@Us6F$4eKIXRzX9B zt7^EK%c?49s<5Sot-QaYf;APctKqr|+iKX(xiuARsBl9KHdM zw~el6MuD5Sng7B#M{a6*9FR3ZQj^lA(0yhSVI(GiUL2*T{v@BE#tC3o=7%P0p!54A zoCMkTcl8~iL|ctlOEqi z)||}O^zW0m3NlIj&`r*qXXecR3z@Vung)S;6qp_@6^`RXZfG1w=5d@b_ND>5EhylJ z%7p@VC3UkZeh%Yc8b;E5k|ln0lJOP4Or|E|jz7z2*=KH$=KU8vbX7?t0QQnN&h}vW zc+ZPR*x=pJKcN@mX!q1jj^bo5|0(%AIny-nI-Zn`s#UGr zvgBRhma!RSzct~JNiKTkpU}R{GsqM;=Ud%!PEA;V%O^u6h3 z+3lBXadr|1X3894qfg@aUbX_pG&Kp@B4oQ?Fna3{YObIepPGX;!%Wj88Tu&7%!x@f zvr>0Ru946TeHBC^mMHKgvyEd%LiB6XEe4gFzr4l)}sK-@Ts0ydGt|H3P0N{xYBO1(xKMY5${v2?XwqhYDBt#B9ts>ZXD8lo}k zC+Js1VMV6TnDIIfNiVLYs~c4rEx>_+S0zsLLJk381?~% zRThSA&W6FD|26$wZ);)tZ!u%8cbU1|{aZB7@)T|b+i>?|DJzcr$kR-JiFeI&M=F+d zbZSD^De67F#rR;b$n9aFRA7~$Z;;#sk@zl-gS^0DP6(uhZu%^r*(8|N@yxM14YH{X zg^%_>5@UF*NtKbFrLGtHk#iilC%x}+dOUeKa$+)vR?AvX@(Y8UO3$YzIn#`F$;4wn z$gs}QnFx>lB+Ue3JQJAANZl%WMbV8PB7oF4CZkoZ=V@+~CJ2aqDXVF%+qbQj-L`JpZMLjy))lK;?b)B$ zJ&WHF^?0`-7#-qC{{g~$dK>QGwDsg#zknY$6PJSlDZmy~|3z}<{l+)?xC@kWAO;|& z?4$56#{-`Pb7XRK06hhY5w?mvV!AvQ7cJz_kNG-0Js#+vfTncvgCOYU=Ro&3PP*Bt z=_1B66Y70PC3p0IElDLwsy@5BB)r3GhCq*7CYhk`u=O#?0f|IC&nXW#(<0&|0HrUt zN_`(ugLkWP(tezTnqVT;vL-%= z*W^V$#eW}yLnX{e@haZbzoG6UkkU!gdY7K!UjH)xOXAmpIgG2Sjgo=iN*n1-cXh0G TrH*f>+G%!boqA`b{^5TCFUSJ1 From 85ad1392f818df26298200ae7a469b47b5400b82 Mon Sep 17 00:00:00 2001 From: saga92 Date: Mon, 9 May 2016 14:11:34 +0800 Subject: [PATCH 4/5] add data migration module --- .gitignore | 1 + Deploy/db/registry.sql | 6 + migration/Dockerfile | 23 ++++ migration/README.md | 51 ++++++++ migration/alembic.sql | 4 + migration/alembic.tpl | 68 ++++++++++ migration/install.sh | 3 + migration/migration.cfg | 4 + migration/migration_harbor/env.py | 85 ++++++++++++ migration/migration_harbor/script.py.mako | 24 ++++ migration/migration_harbor/versions/0_1_1.py | 128 +++++++++++++++++++ migration/run.sh | 94 ++++++++++++++ 12 files changed, 491 insertions(+) create mode 100644 migration/Dockerfile create mode 100644 migration/README.md create mode 100644 migration/alembic.sql create mode 100644 migration/alembic.tpl create mode 100755 migration/install.sh create mode 100644 migration/migration.cfg create mode 100644 migration/migration_harbor/env.py create mode 100644 migration/migration_harbor/script.py.mako create mode 100644 migration/migration_harbor/versions/0_1_1.py create mode 100755 migration/run.sh diff --git a/.gitignore b/.gitignore index 6b04c4ad3..b53274b48 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ Deploy/config/ui/app.conf Deploy/config/db/env Deploy/harbor.cfg ui/ui +*.pyc diff --git a/Deploy/db/registry.sql b/Deploy/db/registry.sql index bd0644b33..1fd0eafa3 100644 --- a/Deploy/db/registry.sql +++ b/Deploy/db/registry.sql @@ -111,3 +111,9 @@ create table properties ( insert into properties (k, v) values ('schema_version', '0.1.1'); + +CREATE TABLE IF NOT EXISTS `alembic_version` ( + `version_num` varchar(32) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +insert into alembic_version values ('0.1.1'); diff --git a/migration/Dockerfile b/migration/Dockerfile new file mode 100644 index 000000000..eb961aac9 --- /dev/null +++ b/migration/Dockerfile @@ -0,0 +1,23 @@ +FROM mysql:5.6 + +MAINTAINER bhe@vmware.com + +RUN sed -i -e 's/us.archive.ubuntu.com/archive.ubuntu.com/g' /etc/apt/sources.list + +RUN apt-get update + +RUN apt-get install -y curl python python-pip git python-mysqldb + +RUN pip install alembic + +RUN mkdir -p /harbor-migration + +WORKDIR /harbor-migration + +COPY ./ ./ + +COPY ./migration.cfg ./ + +RUN ./install.sh + +ENTRYPOINT ["./run.sh"] diff --git a/migration/README.md b/migration/README.md new file mode 100644 index 000000000..1d013e47f --- /dev/null +++ b/migration/README.md @@ -0,0 +1,51 @@ +# migration +Migration is a module for migrating database schema between different version of project [harbor](https://github.com/vmware/harbor) + +**WARNING!!** You must backup your data before migrating + +###installation +- step 1: modify migration.cfg +- step 2: build image from dockerfile + ``` + cd harbor-migration + + docker build -t your-image-name . + ``` + +###migration operation +- show instruction of harbor-migration + + ```docker run your-image-name help``` + +- create backup file in `/path/to/backup` + + ``` + docker run -ti -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup your-image-name backup + ``` + +- restore from backup file in `/path/to/backup` + + ``` + docker run -ti -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup your-image-name restore + ``` + +- perform database schema upgrade + + ```docker run -ti -v /data/database:/var/lib/mysql your-image-name up head``` + +- perform database schema downgrade(downgrade has been disabled) + + ```docker run -v /data/database:/var/lib/mysql your-image-name down base``` + +###migration step +- step 1: stop and remove harbor service + + ``` + docker-compose stop && docker-compose rm -f + ``` +- step 2: perform migration operation +- step 3: rebuild newest harbor images and restart service + + ``` + docker-compose build && docker-compose up -d + ``` diff --git a/migration/alembic.sql b/migration/alembic.sql new file mode 100644 index 000000000..21dc7a1de --- /dev/null +++ b/migration/alembic.sql @@ -0,0 +1,4 @@ +use `registry`; +CREATE TABLE IF NOT EXISTS `alembic_version` ( + `version_num` varchar(32) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/migration/alembic.tpl b/migration/alembic.tpl new file mode 100644 index 000000000..548c6028d --- /dev/null +++ b/migration/alembic.tpl @@ -0,0 +1,68 @@ +echo " +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = migration_harbor + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to migration_harbor/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat migration_harbor/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = mysql://$db_username:$db_password@localhost:$db_port/$db_name + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S" diff --git a/migration/install.sh b/migration/install.sh new file mode 100755 index 000000000..4d707407c --- /dev/null +++ b/migration/install.sh @@ -0,0 +1,3 @@ +#!/bin/bash +source ./migration.cfg +source ./alembic.tpl > ./alembic.ini diff --git a/migration/migration.cfg b/migration/migration.cfg new file mode 100644 index 000000000..a383853ac --- /dev/null +++ b/migration/migration.cfg @@ -0,0 +1,4 @@ +db_username="root" +db_password="root123" +db_port="3306" +db_name="registry" diff --git a/migration/migration_harbor/env.py b/migration/migration_harbor/env.py new file mode 100644 index 000000000..646f39862 --- /dev/null +++ b/migration/migration_harbor/env.py @@ -0,0 +1,85 @@ +# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata + +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migration/migration_harbor/script.py.mako b/migration/migration_harbor/script.py.mako new file mode 100644 index 000000000..43c09401b --- /dev/null +++ b/migration/migration_harbor/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migration/migration_harbor/versions/0_1_1.py b/migration/migration_harbor/versions/0_1_1.py new file mode 100644 index 000000000..0f21b5436 --- /dev/null +++ b/migration/migration_harbor/versions/0_1_1.py @@ -0,0 +1,128 @@ +# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""0.1.0 to 0.1.1 + +Revision ID: 0.1.1 +Revises: +Create Date: 2016-04-18 18:32:14.101897 + +""" + +# revision identifiers, used by Alembic. +revision = '0.1.1' +down_revision = None +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker, relationship +from datetime import datetime + +Session = sessionmaker() + +Base = declarative_base() + +class Properties(Base): + __tablename__ = 'properties' + + k = sa.Column(sa.String(64), primary_key = True) + v = sa.Column(sa.String(128), nullable = False) + +class ProjectMember(Base): + __tablename__ = 'project_member' + + project_id = sa.Column(sa.Integer(), primary_key = True) + user_id = sa.Column(sa.Integer(), primary_key = True) + role = sa.Column(sa.Integer(), nullable = False) + creation_time = sa.Column(sa.DateTime(), nullable = True) + update_time = sa.Column(sa.DateTime(), nullable = True) + sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'], ), + sa.ForeignKeyConstraint(['role'], [u'role.role_id'], ), + sa.ForeignKeyConstraint(['user_id'], [u'user.user_id'], ), + +class UserProjectRole(Base): + __tablename__ = 'user_project_role' + + upr_id = sa.Column(sa.Integer(), primary_key = True) + user_id = sa.Column(sa.Integer(), sa.ForeignKey('user.user_id')) + pr_id = sa.Column(sa.Integer(), sa.ForeignKey('project_role.pr_id')) + project_role = relationship("ProjectRole") + +class ProjectRole(Base): + __tablename__ = 'project_role' + + pr_id = sa.Column(sa.Integer(), primary_key = True) + project_id = sa.Column(sa.Integer(), nullable = False) + role_id = sa.Column(sa.Integer(), nullable = False) + sa.ForeignKeyConstraint(['role_id'], [u'role.role_id']) + sa.ForeignKeyConstraint(['project_id'], [u'project.project_id']) + +class Access(Base): + __tablename__ = 'access' + + access_id = sa.Column(sa.Integer(), primary_key = True) + access_code = sa.Column(sa.String(1)) + comment = sa.Column(sa.String(30)) + +def upgrade(): + """ + update schema&data + """ + bind = op.get_bind() + session = Session(bind=bind) + + #delete M from table access + acc = session.query(Access).filter_by(access_id=1).first() + session.delete(acc) + + #create table property + Properties.__table__.create(bind) + session.add(Properties(k='schema_version', v='0.1.1')) + + #create table project_member + ProjectMember.__table__.create(bind) + + #fill data + join_result = session.query(UserProjectRole).join(UserProjectRole.project_role).all() + for result in join_result: + session.add(ProjectMember(project_id=result.project_role.project_id, \ + user_id=result.user_id, role=result.project_role.role_id, \ + creation_time=datetime.now(), update_time=datetime.now())) + + #drop user_project_role table before drop project_role + #because foreign key constraint + op.drop_table('user_project_role') + op.drop_table('project_role') + + #add column to table project + op.add_column('project', sa.Column('update_time', sa.DateTime(), nullable=True)) + + #add column to table role + op.add_column('role', sa.Column('role_mask', sa.Integer(), server_default=sa.text(u"'0'"), nullable=False)) + + #add column to table user + op.add_column('user', sa.Column('creation_time', sa.DateTime(), nullable=True)) + op.add_column('user', sa.Column('sysadmin_flag', sa.Integer(), nullable=True)) + op.add_column('user', sa.Column('update_time', sa.DateTime(), nullable=True)) + session.commit() + +def downgrade(): + """ + Downgrade has been disabled. + """ + pass diff --git a/migration/run.sh b/migration/run.sh new file mode 100755 index 000000000..806378ff5 --- /dev/null +++ b/migration/run.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +source ./migration.cfg + +WAITTIME=60 + +DBCNF="-hlocalhost -u${db_username}" + +#prevent shell to print insecure message +export MYSQL_PWD="${db_password}" + +if [[ $1 = "help" || $1 = "h" || $# = 0 ]]; then + echo "Usage:" + echo "backup perform database backup" + echo "restore perform database restore" + echo "up, upgrade perform database schema upgrade" + echo "h, help usage help" + exit 0 +fi + +if [[ $1 = "up" || $1 = "upgrade" ]]; then + echo "Please backup before upgrade." + read -p "Enter y to continue updating or n to abort:" ans + case $ans in + [Yy]* ) + ;; + [Nn]* ) + exit 0 + ;; + * ) echo "illegal answer: $ans. Upgrade abort!!" + exit 1 + ;; + esac + +fi + +echo 'Trying to start mysql server...' +DBRUN=0 +nohup mysqld 2>&1 > ./nohup.log& +for i in $(seq 1 $WAITTIME); do + echo "$(/usr/sbin/service mysql status)" + if [[ "$(/usr/sbin/service mysql status)" =~ "not running" ]]; then + sleep 1 + else + DBRUN=1 + break + fi +done + +if [[ $DBRUN -eq 0 ]]; then + echo "timeout. Can't run mysql server." + exit 1 +fi + +key="$1" +case $key in +up|upgrade) + VERSION="$2" + if [[ -z $VERSION ]]; then + VERSION="head" + echo "Version is not specified. Default version is head." + fi + echo "Performing upgrade ${VERSION}..." + if [[ $(mysql $DBCNF -N -s -e "select count(*) from information_schema.tables \ + where table_schema='registry' and table_name='alembic_version';") -eq 0 ]]; then + echo "table alembic_version does not exist. Trying to initial alembic_version." + mysql $DBCNF < ./alembic.sql + #compatible with version 0.1.0 and 0.1.1 + if [[ $(mysql $DBCNF -N -s -e "select count(*) from information_schema.tables \ + where table_schema='registry' and table_name='properties'") -eq 0 ]]; then + echo "table properties does not exist. The version of registry is 0.1.0" + else + echo "The version of registry is 0.1.1" + mysql $DBCNF -e "insert into registry.alembic_version values ('0.1.1')" + fi + fi + alembic -c ./alembic.ini upgrade ${VERSION} + echo "Upgrade performed." + ;; +backup) + echo "Performing backup..." + mysqldump $DBCNF --add-drop-database --databases registry > ./backup/registry.sql + echo "Backup performed." + ;; +restore) + echo "Performing restore..." + mysql $DBCNF < ./backup/registry.sql + echo "Restore performed." + ;; +*) + echo "unknown option" + exit 0 + ;; +esac From 9cf2525c10d178912d306c1fe378bba982404957 Mon Sep 17 00:00:00 2001 From: saga92 Date: Mon, 9 May 2016 18:50:04 +0800 Subject: [PATCH 5/5] fix bugs --- Deploy/prepare | 2 +- migration/Dockerfile | 2 +- migration/{install.sh => prepare.sh} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename migration/{install.sh => prepare.sh} (100%) diff --git a/Deploy/prepare b/Deploy/prepare index 33288d06e..edd31ea95 100755 --- a/Deploy/prepare +++ b/Deploy/prepare @@ -116,7 +116,7 @@ FNULL = open(os.devnull, 'w') from functools import wraps def stat_decorator(func): - #@wraps(func) + @wraps(func) def check_wrapper(*args, **kwargs): stat = func(*args, **kwargs) message = "Generated configuration file: %s" % kwargs['path'] \ diff --git a/migration/Dockerfile b/migration/Dockerfile index eb961aac9..507342170 100644 --- a/migration/Dockerfile +++ b/migration/Dockerfile @@ -18,6 +18,6 @@ COPY ./ ./ COPY ./migration.cfg ./ -RUN ./install.sh +RUN ./prepare.sh ENTRYPOINT ["./run.sh"] diff --git a/migration/install.sh b/migration/prepare.sh similarity index 100% rename from migration/install.sh rename to migration/prepare.sh