From 0517ecca7a6e849a7d1bc9cb742b600e433ef947 Mon Sep 17 00:00:00 2001 From: wang yan Date: Fri, 7 Dec 2018 16:59:25 +0800 Subject: [PATCH] Fix unable to use Notary after updating to v1.6.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit is to fix the issue on notary migrations from mysql to pgsql. 1, alter sequence for the primary key of changeseed, this is missed in v1.6.0 migrator. 2, alter table owners from postgres to notarysigner and notaryserver. Issue: https://github.com/goharbor/harbor/issues/6465 Workaround: https://github.com/goharbor/harbor/issues/6465#issuecomment-445162616 Impacted upgrade path: 1, Upgrade from version older then v1.6.0 with migrator:v1.6.0, and migrates the notarty DB. No impacted upgrade path: 1, Upgrade from version older than v1.6.0 with migrator:v1.6.0, but without migrates the notarty DB. Notes： After merge this fix, we need to provide an new migrator with an new tag, like v1.6.1, and deprecated the v1.6.0. For those who was impacted by migrator v1.6.0, will open an new PR to build the workaround into the migrator and expose an specical command for hot-fix. Signed-off-by: wang yan --- .../db/schema/notaryserver_alter_tables.pgsql | 8 ++++++++ .../db/schema/notaryserver_create_tables.pgsql | 2 +- .../db/schema/notarysigner_alter_tables.pgsql | 5 +++++ .../db/schema/notarysigner_create_tables.pgsql | 2 +- tools/migration/db/util/mysql_pgsql_1_5_0.sh | 3 +++ tools/migration/db/util/mysql_pgsql_data_converter.py | 11 +++++++++-- 6 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 tools/migration/db/schema/notaryserver_alter_tables.pgsql create mode 100644 tools/migration/db/schema/notarysigner_alter_tables.pgsql diff --git a/tools/migration/db/schema/notaryserver_alter_tables.pgsql b/tools/migration/db/schema/notaryserver_alter_tables.pgsql new file mode 100644 index 000000000..5d3370b9f --- /dev/null +++ b/tools/migration/db/schema/notaryserver_alter_tables.pgsql @@ -0,0 +1,8 @@ +\c notaryserver; + +ALTER TABLE tuf_files OWNER TO server; +ALTER SEQUENCE tuf_files_id_seq OWNER TO server; +ALTER TABLE change_category OWNER TO server; +ALTER TABLE changefeed OWNER TO server; +ALTER SEQUENCE changefeed_id_seq OWNER TO server; +ALTER TABLE schema_migrations OWNER TO server; \ No newline at end of file diff --git a/tools/migration/db/schema/notaryserver_create_tables.pgsql b/tools/migration/db/schema/notaryserver_create_tables.pgsql index 0fb7ea6b6..209ba824c 100644 --- a/tools/migration/db/schema/notaryserver_create_tables.pgsql +++ b/tools/migration/db/schema/notaryserver_create_tables.pgsql @@ -1,7 +1,7 @@ \c notaryserver; CREATE TABLE "tuf_files" ( - "id" int PRIMARY KEY, + "id" serial PRIMARY KEY, "created_at" timestamp NULL DEFAULT NULL, "updated_at" timestamp NULL DEFAULT NULL, "deleted_at" timestamp NULL DEFAULT NULL, diff --git a/tools/migration/db/schema/notarysigner_alter_tables.pgsql b/tools/migration/db/schema/notarysigner_alter_tables.pgsql new file mode 100644 index 000000000..19425da20 --- /dev/null +++ b/tools/migration/db/schema/notarysigner_alter_tables.pgsql @@ -0,0 +1,5 @@ +\c notarysigner; + +ALTER TABLE private_keys OWNER TO signer; +ALTER SEQUENCE private_keys_id_seq OWNER TO signer; +ALTER TABLE schema_migrations OWNER TO signer; \ No newline at end of file diff --git a/tools/migration/db/schema/notarysigner_create_tables.pgsql b/tools/migration/db/schema/notarysigner_create_tables.pgsql index ad8cefbe0..0b434c634 100644 --- a/tools/migration/db/schema/notarysigner_create_tables.pgsql +++ b/tools/migration/db/schema/notarysigner_create_tables.pgsql @@ -1,7 +1,7 @@ \c notarysigner; CREATE TABLE "private_keys" ( - "id" int PRIMARY KEY, + "id" serial PRIMARY KEY, "created_at" timestamp NULL DEFAULT NULL, "updated_at" timestamp NULL DEFAULT NULL, "deleted_at" timestamp NULL DEFAULT NULL, diff --git a/tools/migration/db/util/mysql_pgsql_1_5_0.sh b/tools/migration/db/util/mysql_pgsql_1_5_0.sh index 108a4b915..874000548 100644 --- a/tools/migration/db/util/mysql_pgsql_1_5_0.sh +++ b/tools/migration/db/util/mysql_pgsql_1_5_0.sh @@ -73,8 +73,11 @@ EOF # launch_pgsql $PGSQL_USR psql -U $1 -f /harbor-migration/db/schema/notaryserver_create_tables.pgsql psql -U $1 -f /harbor-migration/db/schema/notaryserver_insert_data.pgsql + psql -U $1 -f /harbor-migration/db/schema/notaryserver_alter_tables.pgsql + psql -U $1 -f /harbor-migration/db/schema/notarysigner_create_tables.pgsql psql -U $1 -f /harbor-migration/db/schema/notarysigner_insert_data.pgsql + psql -U $1 -f /harbor-migration/db/schema/notarysigner_alter_tables.pgsql stop_mysql root stop_pgsql $1 diff --git a/tools/migration/db/util/mysql_pgsql_data_converter.py b/tools/migration/db/util/mysql_pgsql_data_converter.py index 716fddced..5a656839f 100644 --- a/tools/migration/db/util/mysql_pgsql_data_converter.py +++ b/tools/migration/db/util/mysql_pgsql_data_converter.py @@ -94,7 +94,8 @@ def convert_notary_server_db(mysql_dump_file, pgsql_dump_file): write_database(pgsql_dump, "notaryserver") write_insert(pgsql_dump, insert_lines) - write_sequence(pgsql_dump, "tuf_files", "id") + alter_sequence(pgsql_dump, "tuf_files", "id") + alter_sequence(pgsql_dump, "changefeed", "id") def convert_notary_signer_db(mysql_dump_file, pgsql_dump_file): mysql_dump = open(mysql_dump_file) @@ -115,7 +116,7 @@ def convert_notary_signer_db(mysql_dump_file, pgsql_dump_file): write_database(pgsql_dump, "notarysigner") write_insert(pgsql_dump, insert_lines) - write_sequence(pgsql_dump, "private_keys", "id") + alter_sequence(pgsql_dump, "private_keys", "id") def write_database(pgsql_dump, db_name): pgsql_dump.write("\\c %s;\n" % db_name) @@ -148,6 +149,12 @@ def write_sequence(pgsql_dump, table_name, table_columnn): pgsql_dump.write("SELECT setval('%s_%s_seq', max(%s)) FROM %s;\n" % (table_name, table_columnn, table_columnn, table_name)) pgsql_dump.write("ALTER TABLE \"%s\" ALTER COLUMN \"%s\" SET DEFAULT nextval('%s_%s_seq');\n" % (table_name, table_columnn, table_name, table_columnn)) +def alter_sequence(pgsql_dump, table_name, table_columnn): + pgsql_dump.write('\n') + pgsql_dump.write("SELECT setval('%s_%s_seq', max(%s)) FROM %s;\n" % (table_name, table_columnn, table_columnn, table_name)) + pgsql_dump.write("ALTER TABLE \"%s\" ALTER COLUMN \"%s\" SET DEFAULT nextval('%s_%s_seq');\n" % (table_name, table_columnn, table_name, table_columnn)) + + if __name__ == "__main__": if sys.argv[1].find("registry") != -1: convert_registry_db(sys.argv[1], sys.argv[2])