From c7a54e115c3f4fae36428ff332b43e81dc0de3ba Mon Sep 17 00:00:00 2001 From: wangyan Date: Fri, 8 Jun 2018 01:15:12 -0700 Subject: [PATCH 1/3] Add clair data migration in DB migrator --- tools/migration/db/run.sh | 15 ++++++++++-- tools/migration/db/util/mysql_pgsql_1_5_0.sh | 11 +++++++++ tools/migration/db/util/pgsql.sh | 24 +++++++++++++------- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/tools/migration/db/run.sh b/tools/migration/db/run.sh index c5310a750..5084e41e5 100755 --- a/tools/migration/db/run.sh +++ b/tools/migration/db/run.sh @@ -23,6 +23,7 @@ set -e ISMYSQL=false ISPGSQL=false ISNOTARY=false +ISCLAIR=false cur_version="" PGSQL_USR="postgres" @@ -45,6 +46,10 @@ function init { ISPGSQL=true fi + if [ -d "/clair-db" ]; then + ISCLAIR=true + fi + if [ $ISMYSQL == false ] && [ $ISPGSQL == false ]; then echo "No database has been mounted for the migration. Use '-v' to set it in 'docker run'." exit 1 @@ -65,7 +70,11 @@ function init { fi if [ $ISPGSQL == true ]; then - launch_pgsql $PGSQL_USR + if [ $ISCLAIR == true ]; then + launch_pgsql $PGSQL_USR "/clair-db" + else + launch_pgsql $PGSQL_USR + fi fi } @@ -130,8 +139,10 @@ function validate { function upgrade { if [ $ISNOTARY == true ];then up_notary $PGSQL_USR + elif [ $ISCLAIR == true ];then + up_clair $PGSQL_USR else - up_harbor $1 + up_harbor $1 fi } diff --git a/tools/migration/db/util/mysql_pgsql_1_5_0.sh b/tools/migration/db/util/mysql_pgsql_1_5_0.sh index 4f626a70d..efff377e7 100644 --- a/tools/migration/db/util/mysql_pgsql_1_5_0.sh +++ b/tools/migration/db/util/mysql_pgsql_1_5_0.sh @@ -79,4 +79,15 @@ EOF stop_mysql root stop_pgsql $1 fi +} + +function up_clair { + # clair DB info: user: 'postgres' database: 'postgres' + pg_dump -U postgres postgres > /harbor-migration/db/schema/clair.pgsql + stop_pgsql postgres "/clair-db" + + # it's harbor DB on pgsql. + launch_pgsql $1 + psql -U $1 -f /harbor-migration/db/schema/clair.pgsql + stop_pgsql $1 } \ No newline at end of file diff --git a/tools/migration/db/util/pgsql.sh b/tools/migration/db/util/pgsql.sh index 0f2cfc34d..e81755d73 100644 --- a/tools/migration/db/util/pgsql.sh +++ b/tools/migration/db/util/pgsql.sh @@ -41,16 +41,20 @@ if [ "${1:0:1}" = '-' ]; then fi function launch_pgsql { + local pg_data=$2 + if [ -z $2 ]; then + pg_data=$PGDATA + fi if [ "$1" = 'postgres' ]; then - chown -R postgres:postgres $PGDATA + chown -R postgres:postgres $pg_data # look specifically for PG_VERSION, as it is expected in the DB dir - if [ ! -s "$PGDATA/PG_VERSION" ]; then + if [ ! -s "$pg_data/PG_VERSION" ]; then file_env 'POSTGRES_INITDB_ARGS' if [ "$POSTGRES_INITDB_XLOGDIR" ]; then export POSTGRES_INITDB_ARGS="$POSTGRES_INITDB_ARGS --xlogdir $POSTGRES_INITDB_XLOGDIR" fi - su - $1 -c "initdb -D $PGDATA -U postgres -E UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8 $POSTGRES_INITDB_ARGS" + su - $1 -c "initdb -D $pg_data -U postgres -E UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8 $POSTGRES_INITDB_ARGS" # check password first so we can output the warning before postgres # messes it up file_env 'POSTGRES_PASSWORD' @@ -66,10 +70,10 @@ function launch_pgsql { { echo echo "host all all all $authMethod" - } >> "$PGDATA/pg_hba.conf" + } >> "$pg_data/pg_hba.conf" # internal start of server in order to allow set-up using psql-client # does not listen on external TCP/IP and waits until start finishes - su - $1 -c "pg_ctl -D \"$PGDATA\" -o \"-c listen_addresses='localhost'\" -w start" + su - $1 -c "pg_ctl -D \"$pg_data\" -o \"-c listen_addresses='localhost'\" -w start" file_env 'POSTGRES_USER' 'postgres' file_env 'POSTGRES_DB' "$POSTGRES_USER" @@ -107,19 +111,23 @@ EOSQL done #PGUSER="${PGUSER:-postgres}" \ - #su - $1 -c "pg_ctl -D \"$PGDATA\" -m fast -w stop" + #su - $1 -c "pg_ctl -D \"$pg_data\" -m fast -w stop" echo echo 'PostgreSQL init process complete; ready for start up.' echo else - su - $PGSQL_USR -c "pg_ctl -D \"$PGDATA\" -o \"-c listen_addresses='localhost'\" -w start" + su - $PGSQL_USR -c "pg_ctl -D \"$pg_data\" -o \"-c listen_addresses='localhost'\" -w start" fi fi } function stop_pgsql { - su - $1 -c "pg_ctl -D \"/var/lib/postgresql/data\" -w stop" + local pg_data=$2 + if [ -z $2 ]; then + pg_data=$PGDATA + fi + su - $1 -c "pg_ctl -D \"$pg_data\" -w stop" } function get_version_pgsql { From 8d4d1582a3ac5b56691f1b75cd1a57d28e02e15e Mon Sep 17 00:00:00 2001 From: wangyan Date: Sun, 10 Jun 2018 20:15:55 -0700 Subject: [PATCH 2/3] add checking for clair DB migration --- tools/migration/db/util/mysql_pgsql_1_5_0.sh | 35 ++++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/tools/migration/db/util/mysql_pgsql_1_5_0.sh b/tools/migration/db/util/mysql_pgsql_1_5_0.sh index efff377e7..5fecdca0a 100644 --- a/tools/migration/db/util/mysql_pgsql_1_5_0.sh +++ b/tools/migration/db/util/mysql_pgsql_1_5_0.sh @@ -83,11 +83,34 @@ EOF function up_clair { # clair DB info: user: 'postgres' database: 'postgres' - pg_dump -U postgres postgres > /harbor-migration/db/schema/clair.pgsql - stop_pgsql postgres "/clair-db" - # it's harbor DB on pgsql. - launch_pgsql $1 - psql -U $1 -f /harbor-migration/db/schema/clair.pgsql - stop_pgsql $1 + set +e + if [[ $(psql -U $1 -d postgres -t -c "select count(*) from vulnerability;") -eq 0 ]]; then + echo "no vulnerability data needs to be updated." + return 0 + else + + ## it's not a clean clair db, so cannot execute the import step. + ## fail at here to call user to clean DB, then to run clair db migration. + if [[ $(psql -U $1 -d postgres -t -c "select count(*) from pg_tables where schemaname='public';") -ne 0 ]]; then + cat >&2 <<-EOF + ******************************************************************************* + WARNING: Clair migration will only allow anyone haven't migrated clair or + launched harbor yet. + If you want to migrate clair data, please delete all the clair DB tables + in pgsql manually fistly. + ******************************************************************************* +EOF + exit 0 + fi + + set -e + pg_dump -U postgres postgres > /harbor-migration/db/schema/clair.pgsql + stop_pgsql postgres "/clair-db" + + # it's harbor DB on pgsql. + launch_pgsql $1 + psql -U $1 -f /harbor-migration/db/schema/clair.pgsql + stop_pgsql $1 + fi } \ No newline at end of file From 2379123ff78e1f7aa6e4004a974522c27129b365 Mon Sep 17 00:00:00 2001 From: wangyan Date: Sun, 10 Jun 2018 23:18:30 -0700 Subject: [PATCH 3/3] fix up clair issue and typo --- tools/migration/db/util/mysql_pgsql_1_5_0.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tools/migration/db/util/mysql_pgsql_1_5_0.sh b/tools/migration/db/util/mysql_pgsql_1_5_0.sh index 5fecdca0a..85e2533b9 100644 --- a/tools/migration/db/util/mysql_pgsql_1_5_0.sh +++ b/tools/migration/db/util/mysql_pgsql_1_5_0.sh @@ -55,7 +55,7 @@ function up_notary { WARNING: Notary migration will only allow anyone haven't migrated notary or launched harbor yet. If you want to migrate notary data, please delete all the notaryserver - and notarysigner DB tables in pgsql manually fistly. + and notarysigner DB tables in pgsql manually firstly. ******************************************************************************* EOF exit 0 @@ -88,8 +88,12 @@ function up_clair { if [[ $(psql -U $1 -d postgres -t -c "select count(*) from vulnerability;") -eq 0 ]]; then echo "no vulnerability data needs to be updated." return 0 - else - + else + pg_dump -U postgres postgres > /harbor-migration/db/schema/clair.pgsql + stop_pgsql postgres "/clair-db" + + # it's harbor DB on pgsql. + launch_pgsql $1 ## it's not a clean clair db, so cannot execute the import step. ## fail at here to call user to clean DB, then to run clair db migration. if [[ $(psql -U $1 -d postgres -t -c "select count(*) from pg_tables where schemaname='public';") -ne 0 ]]; then @@ -98,18 +102,12 @@ function up_clair { WARNING: Clair migration will only allow anyone haven't migrated clair or launched harbor yet. If you want to migrate clair data, please delete all the clair DB tables - in pgsql manually fistly. + in pgsql manually firstly. ******************************************************************************* EOF exit 0 fi - set -e - pg_dump -U postgres postgres > /harbor-migration/db/schema/clair.pgsql - stop_pgsql postgres "/clair-db" - - # it's harbor DB on pgsql. - launch_pgsql $1 psql -U $1 -f /harbor-migration/db/schema/clair.pgsql stop_pgsql $1 fi