Update the migration sql

1. Update the migration sql
2. Rename the ResourceRepository from repository to image

Signed-off-by: Wenkai Yin <yinw@vmware.com>

make/migrations/postgresql/0004_1.8.0_schema.up.sql

@@ -62,15 +62,16 @@ UPDATE registry SET credential_type='basic';
 /*upgrade the replication_policy*/
 ALTER TABLE replication_policy ADD COLUMN creator varchar(256);
 ALTER TABLE replication_policy ADD COLUMN src_registry_id int;
-ALTER TABLE replication_policy ADD COLUMN src_namespaces varchar(256);
+/*The predefined filters will be cleared and replaced by "project_name/"+double star.
-/*if harbor is integrated with the external project service, the src_namespaces will be empty,
+if harbor is integrated with the external project service, we cannot get the project name by ID,
-which means the repilcation policy cannot work as expected*/
+which means the repilcation policy will match all resources.*/
-UPDATE replication_policy r SET src_namespaces=(SELECT p.name FROM project p WHERE p.project_id=r.project_id);
+UPDATE replication_policy r SET filters=(SELECT CONCAT('[{"type":"name","value":"', p.name,'/**"}]') FROM project p WHERE p.project_id=r.project_id);
 ALTER TABLE replication_policy RENAME COLUMN target_id TO dest_registry_id;
 ALTER TABLE replication_policy ALTER COLUMN dest_registry_id DROP NOT NULL;
 ALTER TABLE replication_policy ADD COLUMN dest_namespace varchar(256);
 ALTER TABLE replication_policy ADD COLUMN override boolean;
 ALTER TABLE replication_policy DROP COLUMN project_id;
+ALTER TABLE replication_policy RENAME COLUMN cron_str TO trigger;
 
 DROP TRIGGER replication_immediate_trigger_update_time_at_modtime ON replication_immediate_trigger;
 DROP TABLE replication_immediate_trigger;
@@ -119,10 +120,11 @@ BEGIN
     LOOP
       /*insert one execution record*/
       INSERT INTO replication_execution (policy_id, start_time) VALUES (job.policy_id, job.creation_time) RETURNING id INTO execid;
-      /*insert one task record*/
+      /*insert one task record
+      doesn't record the tags info in "src_resource" and "dst_resource" as the length
+      of the tags may longer than the capability of the column*/
       INSERT INTO replication_task (execution_id, resource_type, src_resource, dst_resource, operation, job_id, status, start_time, end_time) 
-      VALUES (execid, 'image', CONCAT(job.repository,':[', job.tags,']'), CONCAT(job.repository,':[', job.tags,']'),
+      VALUES (execid, 'image', job.repository, job.repository, job.operation, job.job_uuid, job.status, job.creation_time, job.update_time);
-        job.operation, job.job_uuid, job.status, job.creation_time, job.update_time);
     END LOOP;
 END $$;
 UPDATE replication_task SET status='Pending' WHERE status='pending';

src/core/api/repository.go

@@ -331,7 +331,7 @@ func (ra *RepositoryAPI) Delete() {
 			e := &event.Event{
 				Type: event.EventTypeImagePush,
 				Resource: &model.Resource{
-					Type: model.ResourceTypeRepository,
+					Type: model.ResourceTypeImage,
 					Metadata: &model.ResourceMetadata{
 						Repository: &model.Repository{
 							Name: repoName,

src/core/service/notifications/registry/handler.go

@@ -117,7 +117,7 @@ func (n *NotificationHandler) Post() {
 				e := &rep_event.Event{
 					Type: rep_event.EventTypeImagePush,
 					Resource: &model.Resource{
-						Type: model.ResourceTypeRepository,
+						Type: model.ResourceTypeImage,
 						Metadata: &model.ResourceMetadata{
 							Repository: &model.Repository{
 								Name: repository,

src/jobservice/job/impl/replication/replication.go

@@ -25,8 +25,8 @@ import (
 
 	// import chart transfer
 	_ "github.com/goharbor/harbor/src/replication/transfer/chart"
-	// import repository transfer
+	// import image transfer
-	_ "github.com/goharbor/harbor/src/replication/transfer/repository"
+	_ "github.com/goharbor/harbor/src/replication/transfer/image"
 	// register the Harbor adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/harbor"
 	// register the DockerHub adapter

src/replication/adapter/dockerhub/adapter.go

@@ -58,7 +58,7 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
 	return &model.RegistryInfo{
 		Type: model.RegistryTypeDockerHub,
 		SupportedResourceTypes: []model.ResourceType{
-			model.ResourceTypeRepository,
+			model.ResourceTypeImage,
 		},
 		SupportedResourceFilters: []*model.FilterStyle{
 			{
@@ -307,7 +307,7 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
 		}
 
 		resources = append(resources, &model.Resource{
-			Type:     model.ResourceTypeRepository,
+			Type:     model.ResourceTypeImage,
 			Registry: a.registry,
 			Metadata: &model.ResourceMetadata{
 				Repository: &model.Repository{

src/replication/adapter/harbor/adapter.go

@@ -92,7 +92,7 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
 	info := &model.RegistryInfo{
 		Type: model.RegistryTypeHarbor,
 		SupportedResourceTypes: []model.ResourceType{
-			model.ResourceTypeRepository,
+			model.ResourceTypeImage,
 		},
 		SupportedResourceFilters: []*model.FilterStyle{
 			{

src/replication/adapter/harbor/adapter_test.go

@@ -45,7 +45,7 @@ func TestInfo(t *testing.T) {
 	assert.Equal(t, 2, len(info.SupportedResourceFilters))
 	assert.Equal(t, 3, len(info.SupportedTriggers))
 	assert.Equal(t, 2, len(info.SupportedResourceTypes))
-	assert.Equal(t, model.ResourceTypeRepository, info.SupportedResourceTypes[0])
+	assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0])
 	assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[1])
 	server.Close()
 
@@ -69,7 +69,7 @@ func TestInfo(t *testing.T) {
 	assert.Equal(t, 2, len(info.SupportedResourceFilters))
 	assert.Equal(t, 3, len(info.SupportedTriggers))
 	assert.Equal(t, 1, len(info.SupportedResourceTypes))
-	assert.Equal(t, model.ResourceTypeRepository, info.SupportedResourceTypes[0])
+	assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0])
 	server.Close()
 }
 

src/replication/adapter/harbor/image_registry.go

@@ -89,7 +89,7 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
 				vtags = append(vtags, tag.Name)
 			}
 			resources = append(resources, &model.Resource{
-				Type:     model.ResourceTypeRepository,
+				Type:     model.ResourceTypeImage,
 				Registry: a.registry,
 				Metadata: &model.ResourceMetadata{
 					Repository: &model.Repository{

src/replication/adapter/harbor/image_registry_test.go

@@ -70,7 +70,7 @@ func TestFetchImages(t *testing.T) {
 	resources, err := adapter.FetchImages(nil)
 	require.Nil(t, err)
 	assert.Equal(t, 1, len(resources))
-	assert.Equal(t, model.ResourceTypeRepository, resources[0].Type)
+	assert.Equal(t, model.ResourceTypeImage, resources[0].Type)
 	assert.Equal(t, "library/hello-world", resources[0].Metadata.Repository.Name)
 	assert.Equal(t, 2, len(resources[0].Metadata.Vtags))
 	assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0])
@@ -89,7 +89,7 @@ func TestFetchImages(t *testing.T) {
 	resources, err = adapter.FetchImages(filters)
 	require.Nil(t, err)
 	assert.Equal(t, 1, len(resources))
-	assert.Equal(t, model.ResourceTypeRepository, resources[0].Type)
+	assert.Equal(t, model.ResourceTypeImage, resources[0].Type)
 	assert.Equal(t, "library/hello-world", resources[0].Metadata.Repository.Name)
 	assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
 	assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0])

src/replication/adapter/huawei/huawei_adapter.go

@@ -40,7 +40,7 @@ func (adapter Adapter) Info() (*model.RegistryInfo, error) {
 	registryInfo := model.RegistryInfo{
 		Type:                     huawei,
 		Description:              "Adapter for SWR -- The image registry of Huawei Cloud",
-		SupportedResourceTypes:   []model.ResourceType{model.ResourceTypeRepository},
+		SupportedResourceTypes:   []model.ResourceType{model.ResourceTypeImage},
 		SupportedResourceFilters: []*model.FilterStyle{},
 		SupportedTriggers:        []model.TriggerType{},
 	}

src/replication/adapter/huawei/image_registry.go

@@ -96,7 +96,7 @@ func parseRepoQueryResultToResource(repo hwRepoQueryResult) *model.Resource {
 	}
 	resource.Deleted = false
 	resource.Override = false
-	resource.Type = model.ResourceTypeRepository
+	resource.Type = model.ResourceTypeImage
 
 	return &resource
 }

src/replication/adapter/native/adapter.go

@@ -54,7 +54,7 @@ func (native) Info() (info *model.RegistryInfo, err error) {
 	return &model.RegistryInfo{
 		Type: registryTypeNative,
 		SupportedResourceTypes: []model.ResourceType{
-			model.ResourceTypeRepository,
+			model.ResourceTypeImage,
 		},
 		SupportedResourceFilters: []*model.FilterStyle{
 			{

src/replication/adapter/native/adapter_test.go

@@ -47,7 +47,7 @@ func Test_native_Info(t *testing.T) {
 	assert.Equal(t, 1, len(info.SupportedResourceTypes))
 	assert.Equal(t, 2, len(info.SupportedResourceFilters))
 	assert.Equal(t, 2, len(info.SupportedTriggers))
-	assert.Equal(t, model.ResourceTypeRepository, info.SupportedResourceTypes[0])
+	assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0])
 }
 
 func Test_native_PrepareForPush(t *testing.T) {

src/replication/adapter/native/image_registry.go

@@ -52,7 +52,7 @@ func (n native) FetchImages(filters []*model.Filter) ([]*model.Resource, error)
 			continue
 		}
 		resources = append(resources, &model.Resource{
-			Type:     model.ResourceTypeRepository,
+			Type:     model.ResourceTypeImage,
 			Registry: n.registry,
 			Metadata: &model.ResourceMetadata{
 				Repository: &model.Repository{

src/replication/dao/models/policy.go

@@ -4,17 +4,16 @@ import "time"
 
 // RepPolicy is the model for a ng replication policy.
 type RepPolicy struct {
-	ID             int64  `orm:"pk;auto;column(id)" json:"id"`
+	ID                int64     `orm:"pk;auto;column(id)" json:"id"`
-	Name           string `orm:"column(name)" json:"name"`
+	Name              string    `orm:"column(name)" json:"name"`
-	Description    string `orm:"column(description)" json:"description"`
+	Description       string    `orm:"column(description)" json:"description"`
-	Creator        string `orm:"column(creator)" json:"creator"`
+	Creator           string    `orm:"column(creator)" json:"creator"`
-	SrcRegistryID  int64  `orm:"column(src_registry_id)" json:"src_registry_id"`
+	SrcRegistryID     int64     `orm:"column(src_registry_id)" json:"src_registry_id"`
-	DestRegistryID int64  `orm:"column(dest_registry_id)" json:"dest_registry_id"`
+	DestRegistryID    int64     `orm:"column(dest_registry_id)" json:"dest_registry_id"`
-	DestNamespace  string `orm:"column(dest_namespace)" json:"dest_namespace"`
+	DestNamespace     string    `orm:"column(dest_namespace)" json:"dest_namespace"`
-	Override       bool   `orm:"column(override)" json:"override"`
+	Override          bool      `orm:"column(override)" json:"override"`
-	Enabled        bool   `orm:"column(enabled)" json:"enabled"`
+	Enabled           bool      `orm:"column(enabled)" json:"enabled"`
-	// TODO rename the db column to trigger
+	Trigger           string    `orm:"column(trigger)" json:"trigger"`
-	Trigger           string    `orm:"column(cron_str)" json:"trigger"`
 	Filters           string    `orm:"column(filters)" json:"filters"`
 	ReplicateDeletion bool      `orm:"column(replicate_deletion)" json:"replicate_deletion"`
 	CreationTime      time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`

src/replication/model/resource.go

@@ -16,8 +16,8 @@ package model
 
 // the resource type
 const (
-	ResourceTypeRepository ResourceType = "repository"
+	ResourceTypeImage ResourceType = "image"
-	ResourceTypeChart      ResourceType = "chart"
+	ResourceTypeChart ResourceType = "chart"
 )
 
 // ResourceType represents the type of the resource

src/replication/operation/controller_test.go

@@ -123,7 +123,7 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
 	return &model.RegistryInfo{
 		Type: model.RegistryTypeHarbor,
 		SupportedResourceTypes: []model.ResourceType{
-			model.ResourceTypeRepository,
+			model.ResourceTypeImage,
 			model.ResourceTypeChart,
 		},
 		SupportedTriggers: []model.TriggerType{model.TriggerTypeManual},
@@ -139,7 +139,7 @@ func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {
 func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter) ([]*model.Resource, error) {
 	return []*model.Resource{
 		{
-			Type: model.ResourceTypeRepository,
+			Type: model.ResourceTypeImage,
 			Metadata: &model.ResourceMetadata{
 				Repository: &model.Repository{
 					Name: "library/hello-world",
@@ -217,7 +217,7 @@ func TestStartReplication(t *testing.T) {
 		},
 	}
 	resource := &model.Resource{
-		Type: model.ResourceTypeRepository,
+		Type: model.ResourceTypeImage,
 		Metadata: &model.ResourceMetadata{
 			Repository: &model.Repository{
 				Name: "library/hello-world",

src/replication/operation/flow/stage.go

@@ -81,7 +81,7 @@ func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resourc
 	for _, typ := range resTypes {
 		var res []*model.Resource
 		var err error
-		if typ == model.ResourceTypeRepository {
+		if typ == model.ResourceTypeImage {
 			// images
 			reg, ok := adapter.(adp.ImageRegistry)
 			if !ok {

src/replication/operation/flow/stage_test.go

@@ -39,7 +39,7 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
 	return &model.RegistryInfo{
 		Type: model.RegistryTypeHarbor,
 		SupportedResourceTypes: []model.ResourceType{
-			model.ResourceTypeRepository,
+			model.ResourceTypeImage,
 			model.ResourceTypeChart,
 		},
 		SupportedTriggers: []model.TriggerType{model.TriggerTypeManual},
@@ -55,7 +55,7 @@ func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {
 func (f *fakedAdapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
 	return []*model.Resource{
 		{
-			Type: model.ResourceTypeRepository,
+			Type: model.ResourceTypeImage,
 			Metadata: &model.ResourceMetadata{
 				Repository: &model.Repository{
 					Name: "library/hello-world",
@@ -211,7 +211,7 @@ func TestFetchResources(t *testing.T) {
 func TestFilterResources(t *testing.T) {
 	resources := []*model.Resource{
 		{
-			Type: model.ResourceTypeRepository,
+			Type: model.ResourceTypeImage,
 			Metadata: &model.ResourceMetadata{
 				Repository: &model.Repository{
 					Name: "library/hello-world",

src/replication/transfer/image/transfer.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package repository
+package image
 
 import (
 	"errors"
@@ -33,7 +33,7 @@ var (
 )
 
 func init() {
-	if err := trans.RegisterFactory(model.ResourceTypeRepository, factory); err != nil {
+	if err := trans.RegisterFactory(model.ResourceTypeImage, factory); err != nil {
 		log.Errorf("failed to register transfer factory: %v", err)
 	}
 }

src/replication/transfer/image/transfer_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package repository
+package image
 
 import (
 	"bytes"

src/replication/util/util.go

@@ -24,6 +24,9 @@ import (
 
 // Match returns whether the str matches the pattern
 func Match(pattern, str string) (bool, error) {
+	if len(pattern) == 0 {
+		return true, nil
+	}
 	return doublestar.Match(pattern, str)
 }
 

src/replication/util/util_test.go

@@ -29,7 +29,7 @@ func TestMatch(t *testing.T) {
 	}{
 		{
 			pattern: "",
-			str:     "",
+			str:     "library",
 			match:   true,
 		},
 		{