From e8fe2aa60c38e38315c0bb592b32b37205fbbd95 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 5 Apr 2019 12:57:34 +0800 Subject: [PATCH] Upgrade the registry and replication policy tables Upgrade the registry and replication tables in database Signed-off-by: Wenkai Yin --- .../postgresql/0004_1.8.0_schema.up.sql | 74 ++++++++++--------- src/common/dao/dao_test.go | 4 +- src/replication/ng/dao/models/policy.go | 4 +- src/replication/ng/policy/manager/manager.go | 4 + 4 files changed, 46 insertions(+), 40 deletions(-) diff --git a/make/migrations/postgresql/0004_1.8.0_schema.up.sql b/make/migrations/postgresql/0004_1.8.0_schema.up.sql index aa1e55ea2..e5dde644b 100644 --- a/make/migrations/postgresql/0004_1.8.0_schema.up.sql +++ b/make/migrations/postgresql/0004_1.8.0_schema.up.sql @@ -44,42 +44,36 @@ WHERE j.policy_id = p.id AND p.deleted = TRUE; DELETE FROM replication_policy AS p WHERE p.deleted = TRUE; -CREATE TABLE registry ( - id SERIAL PRIMARY KEY NOT NULL, - name varchar(256), - url varchar(256), - credential_type varchar(16), - access_key varchar(128), - access_secret varchar(1024), - type varchar(32), - insecure boolean, - description varchar(1024), - health varchar(16), - creation_time timestamp default CURRENT_TIMESTAMP, - update_time timestamp default CURRENT_TIMESTAMP, - CONSTRAINT unique_registry_name UNIQUE (name) -); +/*upgrade the replication_target to registry*/ +DROP TRIGGER replication_target_update_time_at_modtime ON replication_target; +ALTER TABLE replication_target RENAME TO registry; +ALTER TABLE registry ALTER COLUMN url TYPE varchar(256); +ALTER TABLE registry ADD COLUMN credential_type varchar(16); +UPDATE registry SET credential_type='basic' WHERE credential_type=''; +ALTER TABLE registry RENAME COLUMN username TO access_key; +ALTER TABLE registry RENAME COLUMN password TO access_secret; +ALTER TABLE registry ALTER COLUMN access_secret TYPE varchar(1024); +ALTER TABLE registry ADD COLUMN type varchar(32); +UPDATE registry SET type='harbor' WHERE type=''; +ALTER TABLE registry DROP COLUMN target_type; +ALTER TABLE registry ADD COLUMN description text; +ALTER TABLE registry ADD COLUMN health varchar(16); -CREATE TABLE "replication_policy_ng" ( - "id" SERIAL PRIMARY KEY NOT NULL, - "name" varchar(256), - "description" text, - "creator" varchar(256), - "src_registry_id" int4, - "src_namespaces" varchar(256), - "dest_registry_id" int4, - "dest_namespace" varchar(256), - "override" bool NOT NULL DEFAULT false, - "enabled" bool NOT NULL DEFAULT true, - "cron_str" varchar(256), - "filters" varchar(1024), - "replicate_deletion" bool NOT NULL DEFAULT false, - "start_time" timestamp(6), - "deleted" bool NOT NULL DEFAULT false, - "creation_time" timestamp(6) DEFAULT now(), - "update_time" timestamp(6) DEFAULT now(), - CONSTRAINT unique_policy_ng_name UNIQUE ("name") -); +/*upgrade the replication_policy*/ +ALTER TABLE replication_policy ADD COLUMN creator varchar(256); +ALTER TABLE replication_policy ADD COLUMN src_registry_id int; +ALTER TABLE replication_policy ADD COLUMN src_namespaces varchar(256); +/*if harbor is integrated with the external project service, the src_namespaces will be empty, +which means the repilcation policy cannot work as expected*/ +UPDATE replication_policy r SET src_namespaces=(SELECT p.name FROM project p WHERE p.project_id=r.project_id); +ALTER TABLE replication_policy RENAME COLUMN target_id TO dest_registry_id; +ALTER TABLE replication_policy ALTER COLUMN dest_registry_id DROP NOT NULL; +ALTER TABLE replication_policy ADD COLUMN dest_namespace varchar(256); +ALTER TABLE replication_policy ADD COLUMN override boolean; +ALTER TABLE replication_policy DROP COLUMN project_id; + +DROP TRIGGER replication_immediate_trigger_update_time_at_modtime ON replication_immediate_trigger; +DROP TABLE replication_immediate_trigger; create table replication_execution ( id SERIAL NOT NULL, @@ -122,4 +116,12 @@ create table replication_schedule_job ( update_time timestamp NULL, PRIMARY KEY (id) ); -CREATE INDEX replication_schedule_job_index ON replication_schedule_job (policy_id); \ No newline at end of file +CREATE INDEX replication_schedule_job_index ON replication_schedule_job (policy_id); + +/* + * TODO + * consider how to handle the replication_job; + * the replication_job contains schedule job; + * the schedule job has been removed from jobservice, how to handle this? + * keep consistent with the webhook handler? + */ \ No newline at end of file diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index cb00ab09a..569d56092 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -112,7 +112,7 @@ func cleanByUser(username string) { if err != nil { log.Error(err) } - err = execUpdate(o, `delete from replication_target where id < 99`) + err = execUpdate(o, `delete from registry where id < 99`) if err != nil { log.Error(err) } @@ -164,7 +164,7 @@ func testForAll(m *testing.M) int { func clearAll() { tables := []string{"project_member", "project_metadata", "access_log", "repository", "replication_policy", - "replication_target", "replication_job", "replication_immediate_trigger", "img_scan_job", + "registry", "replication_job", "img_scan_job", "img_scan_overview", "clair_vuln_timestamp", "project", "harbor_user"} for _, t := range tables { if err := ClearTable(t); err != nil { diff --git a/src/replication/ng/dao/models/policy.go b/src/replication/ng/dao/models/policy.go index 5e98524a2..c8657203c 100644 --- a/src/replication/ng/dao/models/policy.go +++ b/src/replication/ng/dao/models/policy.go @@ -21,7 +21,7 @@ type RepPolicy struct { UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` } -// TableName set table name for ORM. TODO: replace _ng suffix. +// TableName set table name for ORM. func (r *RepPolicy) TableName() string { - return "replication_policy_ng" + return "replication_policy" } diff --git a/src/replication/ng/policy/manager/manager.go b/src/replication/ng/policy/manager/manager.go index 7a65423df..980857832 100644 --- a/src/replication/ng/policy/manager/manager.go +++ b/src/replication/ng/policy/manager/manager.go @@ -61,6 +61,10 @@ func convertFromPersistModel(policy *persist_models.RepPolicy) (*model.Policy, e ply.SrcNamespaces = strings.Split(policy.SrcNamespaces, ",") } + // TODO need to consider the consistence with the policies from previous versions + // of Harbor + // both for filter and trigger + // 2. parse Filters if len(policy.Filters) > 0 { filters := []*model.Filter{}