Merge pull request #7327 from ywk253100/190409_upgrade

Handle the policy from previous versions

src/core/api/replication_execution.go

@@ -122,8 +122,8 @@ func (r *ReplicationOperationAPI) CreateExecution() {
 		return
 	}
 
-	trigger := r.GetString("trigger", model.TriggerTypeManual)
+	trigger := r.GetString("trigger", string(model.TriggerTypeManual))
-	executionID, err := ng.OperationCtl.StartReplication(policy, nil, trigger)
+	executionID, err := ng.OperationCtl.StartReplication(policy, nil, model.TriggerType(trigger))
 	if err != nil {
 		r.HandleInternalServerError(fmt.Sprintf("failed to start replication for policy %d: %v", execution.PolicyID, err))
 		return

src/core/api/replication_execution_test.go

@@ -25,7 +25,7 @@ import (
 
 type fakedOperationController struct{}
 
-func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger string) (int64, error) {
+func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) {
 	return 1, nil
 }
 func (f *fakedOperationController) StopReplication(int64) error {

src/replication/ng/dao/models/execution.go

@@ -2,6 +2,8 @@ package models
 
 import (
 	"time"
+
+	"github.com/goharbor/harbor/src/replication/ng/model"
 )
 
 const (
@@ -74,7 +76,7 @@ type Execution struct {
 	Succeed    int               `orm:"column(succeed)" json:"succeed"`
 	InProgress int               `orm:"column(in_progress)" json:"in_progress"`
 	Stopped    int               `orm:"column(stopped)" json:"stopped"`
-	Trigger    string    `orm:"column(trigger)" json:"trigger"`
+	Trigger    model.TriggerType `orm:"column(trigger)" json:"trigger"`
 	StartTime  time.Time         `orm:"column(start_time)" json:"start_time"`
 	EndTime    time.Time         `orm:"column(end_time)" json:"end_time"`
 }

src/replication/ng/event/handler_test.go

@@ -28,7 +28,7 @@ import (
 
 type fakedOperationController struct{}
 
-func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger string) (int64, error) {
+func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) {
 	return 1, nil
 }
 func (f *fakedOperationController) StopReplication(int64) error {

src/replication/ng/model/policy.go

@@ -24,14 +24,14 @@ import (
 
 // const definition
 const (
-	FilterTypeResource = "resource"
+	FilterTypeResource FilterType = "resource"
-	FilterTypeName     = "name"
+	FilterTypeName     FilterType = "name"
-	FilterTypeTag      = "tag"
+	FilterTypeTag      FilterType = "tag"
-	FilterTypeLabel    = "label"
+	FilterTypeLabel    FilterType = "label"
 
-	TriggerTypeManual     = "manual"
+	TriggerTypeManual     TriggerType = "manual"
-	TriggerTypeScheduled  = "scheduled"
+	TriggerTypeScheduled  TriggerType = "scheduled"
-	TriggerTypeEventBased = "event_based"
+	TriggerTypeEventBased TriggerType = "event_based"
 )
 
 // Policy defines the structure of a replication policy

src/replication/ng/operation/controller.go

@@ -33,7 +33,7 @@ import (
 // stop, query, etc.
 type Controller interface {
 	// trigger is used to specified that what this replication is triggered by
-	StartReplication(policy *model.Policy, resource *model.Resource, trigger string) (int64, error)
+	StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error)
 	StopReplication(int64) error
 	ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error)
 	GetExecution(int64) (*models.Execution, error)
@@ -58,7 +58,7 @@ type controller struct {
 	scheduler    scheduler.Scheduler
 }
 
-func (c *controller) StartReplication(policy *model.Policy, resource *model.Resource, trigger string) (int64, error) {
+func (c *controller) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) {
 	if resource != nil && len(resource.Metadata.Vtags) != 1 {
 		return 0, fmt.Errorf("the length of Vtags must be 1: %v", resource.Metadata.Vtags)
 	}
@@ -186,7 +186,7 @@ func (c *controller) GetTaskLog(taskID int64) ([]byte, error) {
 }
 
 // create the execution record in database
-func createExecution(mgr execution.Manager, policyID int64, trigger string) (int64, error) {
+func createExecution(mgr execution.Manager, policyID int64, trigger model.TriggerType) (int64, error) {
 	id, err := mgr.Create(&models.Execution{
 		PolicyID:  policyID,
 		Trigger:   trigger,

src/replication/ng/operation/hook/task_test.go

@@ -28,7 +28,7 @@ type fakedOperationController struct {
 	status string
 }
 
-func (f *fakedOperationController) StartReplication(*model.Policy, *model.Resource, string) (int64, error) {
+func (f *fakedOperationController) StartReplication(*model.Policy, *model.Resource, model.TriggerType) (int64, error) {
 	return 0, nil
 }
 func (f *fakedOperationController) StopReplication(int64) error {

src/replication/ng/policy/manager/manager.go

@@ -17,9 +17,11 @@ package manager
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"strings"
 	"time"
 
+	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/replication/ng/dao"
 	persist_models "github.com/goharbor/harbor/src/replication/ng/dao/models"
 	"github.com/goharbor/harbor/src/replication/ng/model"
@@ -61,34 +63,19 @@ func convertFromPersistModel(policy *persist_models.RepPolicy) (*model.Policy, e
 		ply.SrcNamespaces = strings.Split(policy.SrcNamespaces, ",")
 	}
 
-	// TODO need to consider the consistence with the policies from previous versions
-	// of Harbor
-	// both for filter and trigger
-
 	// 2. parse Filters
-	if len(policy.Filters) > 0 {
+	filters, err := parseFilters(policy.Filters)
-		filters := []*model.Filter{}
+	if err != nil {
-		if err := json.Unmarshal([]byte(policy.Filters), &filters); err != nil {
 		return nil, err
 	}
-		// convert the type of value from string to model.ResourceType if the filter
-		// is a resource type filter
-		for _, filter := range filters {
-			if filter.Type == model.FilterTypeResource {
-				filter.Value = (model.ResourceType)(filter.Value.(string))
-			}
-		}
 	ply.Filters = filters
-	}
 
 	// 3. parse Trigger
-	if len(policy.Trigger) > 0 {
+	trigger, err := parseTrigger(policy.Trigger)
-		trigger := &model.Trigger{}
+	if err != nil {
-		if err := json.Unmarshal([]byte(policy.Trigger), trigger); err != nil {
 		return nil, err
 	}
 	ply.Trigger = trigger
-	}
 
 	return &ply, nil
 }
@@ -216,3 +203,114 @@ func (m *DefaultManager) Update(policy *model.Policy, props ...string) error {
 func (m *DefaultManager) Remove(policyID int64) error {
 	return dao.DeleteRepPolicy(policyID)
 }
+
+type filter struct {
+	Type    model.FilterType `json:"type"`
+	Value   interface{}      `json:"value"`
+	Kind    string           `json:"kind"`
+	Pattern string           `json:"pattern"`
+}
+
+type trigger struct {
+	Type          model.TriggerType      `json:"type"`
+	Settings      *model.TriggerSettings `json:"trigger_settings"`
+	Kind          string                 `json:"kind"`
+	ScheduleParam *scheduleParam         `json:"schedule_param"`
+}
+
+type scheduleParam struct {
+	Type    string `json:"type"`
+	Weekday int8   `json:"weekday"`
+	Offtime int64  `json:"offtime"`
+}
+
+func parseFilters(str string) ([]*model.Filter, error) {
+	if len(str) == 0 {
+		return nil, nil
+	}
+	items := []*filter{}
+	if err := json.Unmarshal([]byte(str), &items); err != nil {
+		return nil, err
+	}
+
+	filters := []*model.Filter{}
+	for _, item := range items {
+		filter := &model.Filter{
+			Type:  item.Type,
+			Value: item.Value,
+		}
+		// keep backwards compatibility
+		if len(filter.Type) == 0 {
+			switch item.Kind {
+			case "repository":
+				filter.Type = model.FilterTypeName
+			case "tag":
+				filter.Type = model.FilterTypeTag
+			case "label":
+				// TODO if we support the label filter, remove the checking logic here
+				continue
+			default:
+				log.Warningf("unknown filter type: %s", filter.Type)
+				continue
+			}
+		}
+		if filter.Value == nil {
+			filter.Value = item.Pattern
+		}
+		// convert the type of value from string to model.ResourceType if the filter
+		// is a resource type filter
+		if filter.Type == model.FilterTypeResource {
+			filter.Value = (model.ResourceType)(filter.Value.(string))
+		}
+		filters = append(filters, filter)
+	}
+	return filters, nil
+}
+
+func parseTrigger(str string) (*model.Trigger, error) {
+	if len(str) == 0 {
+		return nil, nil
+	}
+	item := &trigger{}
+	if err := json.Unmarshal([]byte(str), item); err != nil {
+		return nil, err
+	}
+	trigger := &model.Trigger{
+		Type:     item.Type,
+		Settings: item.Settings,
+	}
+	// keep backwards compatibility
+	if len(trigger.Type) == 0 {
+		switch item.Kind {
+		case "Manual":
+			trigger.Type = model.TriggerTypeManual
+		case "Immediate":
+			trigger.Type = model.TriggerTypeEventBased
+		case "Scheduled":
+			trigger.Type = model.TriggerTypeScheduled
+			trigger.Settings = &model.TriggerSettings{
+				Cron: parseScheduleParamToCron(item.ScheduleParam),
+			}
+		default:
+			log.Warningf("unknown trigger type: %s", item.Kind)
+			return nil, nil
+		}
+	}
+	return trigger, nil
+}
+
+func parseScheduleParamToCron(param *scheduleParam) string {
+	if param == nil {
+		return ""
+	}
+	offtime := param.Offtime
+	offtime = offtime % (3600 * 24)
+	hour := int(offtime / 3600)
+	offtime = offtime % 3600
+	minute := int(offtime / 60)
+	second := int(offtime % 60)
+	if param.Type == "Weekly" {
+		return fmt.Sprintf("%d %d %d * * %d", second, minute, hour, param.Weekday%7)
+	}
+	return fmt.Sprintf("%d %d %d * * *", second, minute, hour)
+}

src/replication/ng/policy/manager/manager_test.go

@@ -203,3 +203,70 @@ func TestNewDefaultManager(t *testing.T) {
 		})
 	}
 }
+
+func TestParseFilters(t *testing.T) {
+	// nil filter string
+	str := ""
+	filters, err := parseFilters(str)
+	require.Nil(t, err)
+	assert.Nil(t, filters)
+	// only contains the fields that introduced in the latest version
+	str = `[{"type":"name","value":"library/hello-world"}]`
+	filters, err = parseFilters(str)
+	require.Nil(t, err)
+	require.Equal(t, 1, len(filters))
+	assert.Equal(t, model.FilterTypeName, filters[0].Type)
+	assert.Equal(t, "library/hello-world", filters[0].Value.(string))
+	// contains "kind" from previous versions
+	str = `[{"kind":"repository","value":"library/hello-world"}]`
+	filters, err = parseFilters(str)
+	require.Nil(t, err)
+	require.Equal(t, 1, len(filters))
+	assert.Equal(t, model.FilterTypeName, filters[0].Type)
+	assert.Equal(t, "library/hello-world", filters[0].Value.(string))
+	// contains "pattern" from previous versions
+	str = `[{"kind":"repository","pattern":"library/hello-world"}]`
+	filters, err = parseFilters(str)
+	require.Nil(t, err)
+	require.Equal(t, 1, len(filters))
+	assert.Equal(t, model.FilterTypeName, filters[0].Type)
+	assert.Equal(t, "library/hello-world", filters[0].Value.(string))
+}
+
+func TestParseTrigger(t *testing.T) {
+	// nil trigger string
+	str := ""
+	trigger, err := parseTrigger(str)
+	require.Nil(t, err)
+	assert.Nil(t, trigger)
+	// only contains the fields that introduced in the latest version
+	str = `{"type":"scheduled", "trigger_settings":{"cron":"1 * * * * *"}}`
+	trigger, err = parseTrigger(str)
+	require.Nil(t, err)
+	assert.Equal(t, model.TriggerTypeScheduled, trigger.Type)
+	assert.Equal(t, "1 * * * * *", trigger.Settings.Cron)
+	// contains "kind" from previous versions
+	str = `{"kind":"Manual"}`
+	trigger, err = parseTrigger(str)
+	require.Nil(t, err)
+	assert.Equal(t, model.TriggerTypeManual, trigger.Type)
+	assert.Nil(t, trigger.Settings)
+	// contains "kind" from previous versions
+	str = `{"kind":"Immediate"}`
+	trigger, err = parseTrigger(str)
+	require.Nil(t, err)
+	assert.Equal(t, model.TriggerTypeEventBased, trigger.Type)
+	assert.Nil(t, trigger.Settings)
+	// contains "schedule_param" from previous versions
+	str = `{"kind":"Scheduled","schedule_param":{"type":"Weekly","weekday":1,"offtime":0}}`
+	trigger, err = parseTrigger(str)
+	require.Nil(t, err)
+	assert.Equal(t, model.TriggerTypeScheduled, trigger.Type)
+	assert.Equal(t, "0 0 0 * * 1", trigger.Settings.Cron)
+	// contains "schedule_param" from previous versions
+	str = `{"kind":"Scheduled","schedule_param":{"type":"Daily","offtime":0}}`
+	trigger, err = parseTrigger(str)
+	require.Nil(t, err)
+	assert.Equal(t, model.TriggerTypeScheduled, trigger.Type)
+	assert.Equal(t, "0 0 0 * * *", trigger.Settings.Cron)
+}