diff --git a/make/migrations/postgresql/0004_1.8.0_schema.up.sql b/make/migrations/postgresql/0004_1.8.0_schema.up.sql index 94c7e6a26..8d631c0e7 100644 --- a/make/migrations/postgresql/0004_1.8.0_schema.up.sql +++ b/make/migrations/postgresql/0004_1.8.0_schema.up.sql @@ -62,15 +62,16 @@ UPDATE registry SET credential_type='basic'; /*upgrade the replication_policy*/ ALTER TABLE replication_policy ADD COLUMN creator varchar(256); ALTER TABLE replication_policy ADD COLUMN src_registry_id int; -ALTER TABLE replication_policy ADD COLUMN src_namespaces varchar(256); -/*if harbor is integrated with the external project service, the src_namespaces will be empty, -which means the repilcation policy cannot work as expected*/ -UPDATE replication_policy r SET src_namespaces=(SELECT p.name FROM project p WHERE p.project_id=r.project_id); +/*The predefined filters will be cleared and replaced by "project_name/"+double star. +if harbor is integrated with the external project service, we cannot get the project name by ID, +which means the repilcation policy will match all resources.*/ +UPDATE replication_policy r SET filters=(SELECT CONCAT('[{"type":"name","value":"', p.name,'/**"}]') FROM project p WHERE p.project_id=r.project_id); ALTER TABLE replication_policy RENAME COLUMN target_id TO dest_registry_id; ALTER TABLE replication_policy ALTER COLUMN dest_registry_id DROP NOT NULL; ALTER TABLE replication_policy ADD COLUMN dest_namespace varchar(256); ALTER TABLE replication_policy ADD COLUMN override boolean; ALTER TABLE replication_policy DROP COLUMN project_id; +ALTER TABLE replication_policy RENAME COLUMN cron_str TO trigger; DROP TRIGGER replication_immediate_trigger_update_time_at_modtime ON replication_immediate_trigger; DROP TABLE replication_immediate_trigger; @@ -119,10 +120,11 @@ BEGIN LOOP /*insert one execution record*/ INSERT INTO replication_execution (policy_id, start_time) VALUES (job.policy_id, job.creation_time) RETURNING id INTO execid; - /*insert one task record*/ + /*insert one task record + doesn't record the tags info in "src_resource" and "dst_resource" as the length + of the tags may longer than the capability of the column*/ INSERT INTO replication_task (execution_id, resource_type, src_resource, dst_resource, operation, job_id, status, start_time, end_time) - VALUES (execid, 'image', CONCAT(job.repository,':[', job.tags,']'), CONCAT(job.repository,':[', job.tags,']'), - job.operation, job.job_uuid, job.status, job.creation_time, job.update_time); + VALUES (execid, 'image', job.repository, job.repository, job.operation, job.job_uuid, job.status, job.creation_time, job.update_time); END LOOP; END $$; UPDATE replication_task SET status='Pending' WHERE status='pending'; diff --git a/src/core/api/repository.go b/src/core/api/repository.go index af281fda8..3fee49730 100644 --- a/src/core/api/repository.go +++ b/src/core/api/repository.go @@ -331,7 +331,7 @@ func (ra *RepositoryAPI) Delete() { e := &event.Event{ Type: event.EventTypeImagePush, Resource: &model.Resource{ - Type: model.ResourceTypeRepository, + Type: model.ResourceTypeImage, Metadata: &model.ResourceMetadata{ Repository: &model.Repository{ Name: repoName, diff --git a/src/core/service/notifications/registry/handler.go b/src/core/service/notifications/registry/handler.go index a9cc1de6a..16f8f2585 100644 --- a/src/core/service/notifications/registry/handler.go +++ b/src/core/service/notifications/registry/handler.go @@ -117,7 +117,7 @@ func (n *NotificationHandler) Post() { e := &rep_event.Event{ Type: rep_event.EventTypeImagePush, Resource: &model.Resource{ - Type: model.ResourceTypeRepository, + Type: model.ResourceTypeImage, Metadata: &model.ResourceMetadata{ Repository: &model.Repository{ Name: repository, diff --git a/src/jobservice/job/impl/replication/replication.go b/src/jobservice/job/impl/replication/replication.go index 70d748862..e200a7048 100644 --- a/src/jobservice/job/impl/replication/replication.go +++ b/src/jobservice/job/impl/replication/replication.go @@ -25,8 +25,8 @@ import ( // import chart transfer _ "github.com/goharbor/harbor/src/replication/transfer/chart" - // import repository transfer - _ "github.com/goharbor/harbor/src/replication/transfer/repository" + // import image transfer + _ "github.com/goharbor/harbor/src/replication/transfer/image" // register the Harbor adapter _ "github.com/goharbor/harbor/src/replication/adapter/harbor" // register the DockerHub adapter diff --git a/src/replication/adapter/dockerhub/adapter.go b/src/replication/adapter/dockerhub/adapter.go index 9d5b27b7c..1be62b74b 100644 --- a/src/replication/adapter/dockerhub/adapter.go +++ b/src/replication/adapter/dockerhub/adapter.go @@ -58,7 +58,7 @@ func (a *adapter) Info() (*model.RegistryInfo, error) { return &model.RegistryInfo{ Type: model.RegistryTypeDockerHub, SupportedResourceTypes: []model.ResourceType{ - model.ResourceTypeRepository, + model.ResourceTypeImage, }, SupportedResourceFilters: []*model.FilterStyle{ { @@ -307,7 +307,7 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error } resources = append(resources, &model.Resource{ - Type: model.ResourceTypeRepository, + Type: model.ResourceTypeImage, Registry: a.registry, Metadata: &model.ResourceMetadata{ Repository: &model.Repository{ diff --git a/src/replication/adapter/harbor/adapter.go b/src/replication/adapter/harbor/adapter.go index b3d9afc37..dbe832d51 100644 --- a/src/replication/adapter/harbor/adapter.go +++ b/src/replication/adapter/harbor/adapter.go @@ -92,7 +92,7 @@ func (a *adapter) Info() (*model.RegistryInfo, error) { info := &model.RegistryInfo{ Type: model.RegistryTypeHarbor, SupportedResourceTypes: []model.ResourceType{ - model.ResourceTypeRepository, + model.ResourceTypeImage, }, SupportedResourceFilters: []*model.FilterStyle{ { diff --git a/src/replication/adapter/harbor/adapter_test.go b/src/replication/adapter/harbor/adapter_test.go index 94e201d1d..b260e5231 100644 --- a/src/replication/adapter/harbor/adapter_test.go +++ b/src/replication/adapter/harbor/adapter_test.go @@ -45,7 +45,7 @@ func TestInfo(t *testing.T) { assert.Equal(t, 2, len(info.SupportedResourceFilters)) assert.Equal(t, 3, len(info.SupportedTriggers)) assert.Equal(t, 2, len(info.SupportedResourceTypes)) - assert.Equal(t, model.ResourceTypeRepository, info.SupportedResourceTypes[0]) + assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0]) assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[1]) server.Close() @@ -69,7 +69,7 @@ func TestInfo(t *testing.T) { assert.Equal(t, 2, len(info.SupportedResourceFilters)) assert.Equal(t, 3, len(info.SupportedTriggers)) assert.Equal(t, 1, len(info.SupportedResourceTypes)) - assert.Equal(t, model.ResourceTypeRepository, info.SupportedResourceTypes[0]) + assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0]) server.Close() } diff --git a/src/replication/adapter/harbor/image_registry.go b/src/replication/adapter/harbor/image_registry.go index ee1d1e015..b982262c8 100644 --- a/src/replication/adapter/harbor/image_registry.go +++ b/src/replication/adapter/harbor/image_registry.go @@ -89,7 +89,7 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error vtags = append(vtags, tag.Name) } resources = append(resources, &model.Resource{ - Type: model.ResourceTypeRepository, + Type: model.ResourceTypeImage, Registry: a.registry, Metadata: &model.ResourceMetadata{ Repository: &model.Repository{ diff --git a/src/replication/adapter/harbor/image_registry_test.go b/src/replication/adapter/harbor/image_registry_test.go index 51449d2a8..eeec13fe8 100644 --- a/src/replication/adapter/harbor/image_registry_test.go +++ b/src/replication/adapter/harbor/image_registry_test.go @@ -70,7 +70,7 @@ func TestFetchImages(t *testing.T) { resources, err := adapter.FetchImages(nil) require.Nil(t, err) assert.Equal(t, 1, len(resources)) - assert.Equal(t, model.ResourceTypeRepository, resources[0].Type) + assert.Equal(t, model.ResourceTypeImage, resources[0].Type) assert.Equal(t, "library/hello-world", resources[0].Metadata.Repository.Name) assert.Equal(t, 2, len(resources[0].Metadata.Vtags)) assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0]) @@ -89,7 +89,7 @@ func TestFetchImages(t *testing.T) { resources, err = adapter.FetchImages(filters) require.Nil(t, err) assert.Equal(t, 1, len(resources)) - assert.Equal(t, model.ResourceTypeRepository, resources[0].Type) + assert.Equal(t, model.ResourceTypeImage, resources[0].Type) assert.Equal(t, "library/hello-world", resources[0].Metadata.Repository.Name) assert.Equal(t, 1, len(resources[0].Metadata.Vtags)) assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0]) diff --git a/src/replication/adapter/huawei/huawei_adapter.go b/src/replication/adapter/huawei/huawei_adapter.go index 7deca8b89..256020bea 100644 --- a/src/replication/adapter/huawei/huawei_adapter.go +++ b/src/replication/adapter/huawei/huawei_adapter.go @@ -40,7 +40,7 @@ func (adapter Adapter) Info() (*model.RegistryInfo, error) { registryInfo := model.RegistryInfo{ Type: huawei, Description: "Adapter for SWR -- The image registry of Huawei Cloud", - SupportedResourceTypes: []model.ResourceType{model.ResourceTypeRepository}, + SupportedResourceTypes: []model.ResourceType{model.ResourceTypeImage}, SupportedResourceFilters: []*model.FilterStyle{}, SupportedTriggers: []model.TriggerType{}, } diff --git a/src/replication/adapter/huawei/image_registry.go b/src/replication/adapter/huawei/image_registry.go index 63e0658eb..f37788ccd 100644 --- a/src/replication/adapter/huawei/image_registry.go +++ b/src/replication/adapter/huawei/image_registry.go @@ -96,7 +96,7 @@ func parseRepoQueryResultToResource(repo hwRepoQueryResult) *model.Resource { } resource.Deleted = false resource.Override = false - resource.Type = model.ResourceTypeRepository + resource.Type = model.ResourceTypeImage return &resource } diff --git a/src/replication/adapter/native/adapter.go b/src/replication/adapter/native/adapter.go index e4ffbbed5..aecac52f3 100644 --- a/src/replication/adapter/native/adapter.go +++ b/src/replication/adapter/native/adapter.go @@ -54,7 +54,7 @@ func (native) Info() (info *model.RegistryInfo, err error) { return &model.RegistryInfo{ Type: registryTypeNative, SupportedResourceTypes: []model.ResourceType{ - model.ResourceTypeRepository, + model.ResourceTypeImage, }, SupportedResourceFilters: []*model.FilterStyle{ { diff --git a/src/replication/adapter/native/adapter_test.go b/src/replication/adapter/native/adapter_test.go index d0f0cb642..c2dd8e6a5 100644 --- a/src/replication/adapter/native/adapter_test.go +++ b/src/replication/adapter/native/adapter_test.go @@ -47,7 +47,7 @@ func Test_native_Info(t *testing.T) { assert.Equal(t, 1, len(info.SupportedResourceTypes)) assert.Equal(t, 2, len(info.SupportedResourceFilters)) assert.Equal(t, 2, len(info.SupportedTriggers)) - assert.Equal(t, model.ResourceTypeRepository, info.SupportedResourceTypes[0]) + assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0]) } func Test_native_PrepareForPush(t *testing.T) { diff --git a/src/replication/adapter/native/image_registry.go b/src/replication/adapter/native/image_registry.go index 1dda3f93c..44a5510fa 100644 --- a/src/replication/adapter/native/image_registry.go +++ b/src/replication/adapter/native/image_registry.go @@ -52,7 +52,7 @@ func (n native) FetchImages(filters []*model.Filter) ([]*model.Resource, error) continue } resources = append(resources, &model.Resource{ - Type: model.ResourceTypeRepository, + Type: model.ResourceTypeImage, Registry: n.registry, Metadata: &model.ResourceMetadata{ Repository: &model.Repository{ diff --git a/src/replication/dao/models/policy.go b/src/replication/dao/models/policy.go index 466ea06d6..6f96e1c17 100644 --- a/src/replication/dao/models/policy.go +++ b/src/replication/dao/models/policy.go @@ -4,17 +4,16 @@ import "time" // RepPolicy is the model for a ng replication policy. type RepPolicy struct { - ID int64 `orm:"pk;auto;column(id)" json:"id"` - Name string `orm:"column(name)" json:"name"` - Description string `orm:"column(description)" json:"description"` - Creator string `orm:"column(creator)" json:"creator"` - SrcRegistryID int64 `orm:"column(src_registry_id)" json:"src_registry_id"` - DestRegistryID int64 `orm:"column(dest_registry_id)" json:"dest_registry_id"` - DestNamespace string `orm:"column(dest_namespace)" json:"dest_namespace"` - Override bool `orm:"column(override)" json:"override"` - Enabled bool `orm:"column(enabled)" json:"enabled"` - // TODO rename the db column to trigger - Trigger string `orm:"column(cron_str)" json:"trigger"` + ID int64 `orm:"pk;auto;column(id)" json:"id"` + Name string `orm:"column(name)" json:"name"` + Description string `orm:"column(description)" json:"description"` + Creator string `orm:"column(creator)" json:"creator"` + SrcRegistryID int64 `orm:"column(src_registry_id)" json:"src_registry_id"` + DestRegistryID int64 `orm:"column(dest_registry_id)" json:"dest_registry_id"` + DestNamespace string `orm:"column(dest_namespace)" json:"dest_namespace"` + Override bool `orm:"column(override)" json:"override"` + Enabled bool `orm:"column(enabled)" json:"enabled"` + Trigger string `orm:"column(trigger)" json:"trigger"` Filters string `orm:"column(filters)" json:"filters"` ReplicateDeletion bool `orm:"column(replicate_deletion)" json:"replicate_deletion"` CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` diff --git a/src/replication/model/resource.go b/src/replication/model/resource.go index f53be4133..23125074c 100644 --- a/src/replication/model/resource.go +++ b/src/replication/model/resource.go @@ -16,8 +16,8 @@ package model // the resource type const ( - ResourceTypeRepository ResourceType = "repository" - ResourceTypeChart ResourceType = "chart" + ResourceTypeImage ResourceType = "image" + ResourceTypeChart ResourceType = "chart" ) // ResourceType represents the type of the resource diff --git a/src/replication/operation/controller_test.go b/src/replication/operation/controller_test.go index b379010c2..a0b95f280 100644 --- a/src/replication/operation/controller_test.go +++ b/src/replication/operation/controller_test.go @@ -123,7 +123,7 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) { return &model.RegistryInfo{ Type: model.RegistryTypeHarbor, SupportedResourceTypes: []model.ResourceType{ - model.ResourceTypeRepository, + model.ResourceTypeImage, model.ResourceTypeChart, }, SupportedTriggers: []model.TriggerType{model.TriggerTypeManual}, @@ -139,7 +139,7 @@ func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) { func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter) ([]*model.Resource, error) { return []*model.Resource{ { - Type: model.ResourceTypeRepository, + Type: model.ResourceTypeImage, Metadata: &model.ResourceMetadata{ Repository: &model.Repository{ Name: "library/hello-world", @@ -217,7 +217,7 @@ func TestStartReplication(t *testing.T) { }, } resource := &model.Resource{ - Type: model.ResourceTypeRepository, + Type: model.ResourceTypeImage, Metadata: &model.ResourceMetadata{ Repository: &model.Repository{ Name: "library/hello-world", diff --git a/src/replication/operation/flow/stage.go b/src/replication/operation/flow/stage.go index 268699bf9..99774ecfc 100644 --- a/src/replication/operation/flow/stage.go +++ b/src/replication/operation/flow/stage.go @@ -81,7 +81,7 @@ func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resourc for _, typ := range resTypes { var res []*model.Resource var err error - if typ == model.ResourceTypeRepository { + if typ == model.ResourceTypeImage { // images reg, ok := adapter.(adp.ImageRegistry) if !ok { diff --git a/src/replication/operation/flow/stage_test.go b/src/replication/operation/flow/stage_test.go index d448d1524..1dd8a7498 100644 --- a/src/replication/operation/flow/stage_test.go +++ b/src/replication/operation/flow/stage_test.go @@ -39,7 +39,7 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) { return &model.RegistryInfo{ Type: model.RegistryTypeHarbor, SupportedResourceTypes: []model.ResourceType{ - model.ResourceTypeRepository, + model.ResourceTypeImage, model.ResourceTypeChart, }, SupportedTriggers: []model.TriggerType{model.TriggerTypeManual}, @@ -55,7 +55,7 @@ func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) { func (f *fakedAdapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) { return []*model.Resource{ { - Type: model.ResourceTypeRepository, + Type: model.ResourceTypeImage, Metadata: &model.ResourceMetadata{ Repository: &model.Repository{ Name: "library/hello-world", @@ -211,7 +211,7 @@ func TestFetchResources(t *testing.T) { func TestFilterResources(t *testing.T) { resources := []*model.Resource{ { - Type: model.ResourceTypeRepository, + Type: model.ResourceTypeImage, Metadata: &model.ResourceMetadata{ Repository: &model.Repository{ Name: "library/hello-world", diff --git a/src/replication/transfer/repository/transfer.go b/src/replication/transfer/image/transfer.go similarity index 98% rename from src/replication/transfer/repository/transfer.go rename to src/replication/transfer/image/transfer.go index 472a9c904..89dddf902 100644 --- a/src/replication/transfer/repository/transfer.go +++ b/src/replication/transfer/image/transfer.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package repository +package image import ( "errors" @@ -33,7 +33,7 @@ var ( ) func init() { - if err := trans.RegisterFactory(model.ResourceTypeRepository, factory); err != nil { + if err := trans.RegisterFactory(model.ResourceTypeImage, factory); err != nil { log.Errorf("failed to register transfer factory: %v", err) } } diff --git a/src/replication/transfer/repository/transfer_test.go b/src/replication/transfer/image/transfer_test.go similarity index 99% rename from src/replication/transfer/repository/transfer_test.go rename to src/replication/transfer/image/transfer_test.go index 0e30db503..2e70b12fd 100644 --- a/src/replication/transfer/repository/transfer_test.go +++ b/src/replication/transfer/image/transfer_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package repository +package image import ( "bytes" diff --git a/src/replication/util/util.go b/src/replication/util/util.go index 6441f1da7..f2fae3994 100644 --- a/src/replication/util/util.go +++ b/src/replication/util/util.go @@ -24,6 +24,9 @@ import ( // Match returns whether the str matches the pattern func Match(pattern, str string) (bool, error) { + if len(pattern) == 0 { + return true, nil + } return doublestar.Match(pattern, str) } diff --git a/src/replication/util/util_test.go b/src/replication/util/util_test.go index 500f24b5c..47721a622 100644 --- a/src/replication/util/util_test.go +++ b/src/replication/util/util_test.go @@ -29,7 +29,7 @@ func TestMatch(t *testing.T) { }{ { pattern: "", - str: "", + str: "library", match: true, }, {