From 5a65480594e9b59ee427e630627b86fb48521984 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Tue, 9 Apr 2019 15:49:36 +0800 Subject: [PATCH] Handle the policy from previous versions Handle the policy from previous versions Signed-off-by: Wenkai Yin --- src/core/api/replication_execution.go | 4 +- src/core/api/replication_execution_test.go | 2 +- src/replication/ng/dao/models/execution.go | 26 ++-- src/replication/ng/event/handler_test.go | 2 +- src/replication/ng/model/policy.go | 14 +- src/replication/ng/operation/controller.go | 6 +- .../ng/operation/hook/task_test.go | 2 +- src/replication/ng/policy/manager/manager.go | 144 +++++++++++++++--- .../ng/policy/manager/manager_test.go | 67 ++++++++ 9 files changed, 217 insertions(+), 50 deletions(-) diff --git a/src/core/api/replication_execution.go b/src/core/api/replication_execution.go index aab260e4d..47bd34a92 100644 --- a/src/core/api/replication_execution.go +++ b/src/core/api/replication_execution.go @@ -122,8 +122,8 @@ func (r *ReplicationOperationAPI) CreateExecution() { return } - trigger := r.GetString("trigger", model.TriggerTypeManual) - executionID, err := ng.OperationCtl.StartReplication(policy, nil, trigger) + trigger := r.GetString("trigger", string(model.TriggerTypeManual)) + executionID, err := ng.OperationCtl.StartReplication(policy, nil, model.TriggerType(trigger)) if err != nil { r.HandleInternalServerError(fmt.Sprintf("failed to start replication for policy %d: %v", execution.PolicyID, err)) return diff --git a/src/core/api/replication_execution_test.go b/src/core/api/replication_execution_test.go index 2524c1405..d8f5cc054 100644 --- a/src/core/api/replication_execution_test.go +++ b/src/core/api/replication_execution_test.go @@ -25,7 +25,7 @@ import ( type fakedOperationController struct{} -func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger string) (int64, error) { +func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) { return 1, nil } func (f *fakedOperationController) StopReplication(int64) error { diff --git a/src/replication/ng/dao/models/execution.go b/src/replication/ng/dao/models/execution.go index 15119be4f..56eae10ff 100644 --- a/src/replication/ng/dao/models/execution.go +++ b/src/replication/ng/dao/models/execution.go @@ -2,6 +2,8 @@ package models import ( "time" + + "github.com/goharbor/harbor/src/replication/ng/model" ) const ( @@ -65,18 +67,18 @@ type ExecutionFieldsName struct { // Execution holds information about once replication execution. type Execution struct { - ID int64 `orm:"pk;auto;column(id)" json:"id"` - PolicyID int64 `orm:"column(policy_id)" json:"policy_id"` - Status string `orm:"column(status)" json:"status"` - StatusText string `orm:"column(status_text)" json:"status_text"` - Total int `orm:"column(total)" json:"total"` - Failed int `orm:"column(failed)" json:"failed"` - Succeed int `orm:"column(succeed)" json:"succeed"` - InProgress int `orm:"column(in_progress)" json:"in_progress"` - Stopped int `orm:"column(stopped)" json:"stopped"` - Trigger string `orm:"column(trigger)" json:"trigger"` - StartTime time.Time `orm:"column(start_time)" json:"start_time"` - EndTime time.Time `orm:"column(end_time)" json:"end_time"` + ID int64 `orm:"pk;auto;column(id)" json:"id"` + PolicyID int64 `orm:"column(policy_id)" json:"policy_id"` + Status string `orm:"column(status)" json:"status"` + StatusText string `orm:"column(status_text)" json:"status_text"` + Total int `orm:"column(total)" json:"total"` + Failed int `orm:"column(failed)" json:"failed"` + Succeed int `orm:"column(succeed)" json:"succeed"` + InProgress int `orm:"column(in_progress)" json:"in_progress"` + Stopped int `orm:"column(stopped)" json:"stopped"` + Trigger model.TriggerType `orm:"column(trigger)" json:"trigger"` + StartTime time.Time `orm:"column(start_time)" json:"start_time"` + EndTime time.Time `orm:"column(end_time)" json:"end_time"` } // TaskPropsName defines the names of fields of Task diff --git a/src/replication/ng/event/handler_test.go b/src/replication/ng/event/handler_test.go index 3c189e80b..59a26e153 100644 --- a/src/replication/ng/event/handler_test.go +++ b/src/replication/ng/event/handler_test.go @@ -28,7 +28,7 @@ import ( type fakedOperationController struct{} -func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger string) (int64, error) { +func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) { return 1, nil } func (f *fakedOperationController) StopReplication(int64) error { diff --git a/src/replication/ng/model/policy.go b/src/replication/ng/model/policy.go index 8ba0b10b6..ccf0b824f 100644 --- a/src/replication/ng/model/policy.go +++ b/src/replication/ng/model/policy.go @@ -24,14 +24,14 @@ import ( // const definition const ( - FilterTypeResource = "resource" - FilterTypeName = "name" - FilterTypeTag = "tag" - FilterTypeLabel = "label" + FilterTypeResource FilterType = "resource" + FilterTypeName FilterType = "name" + FilterTypeTag FilterType = "tag" + FilterTypeLabel FilterType = "label" - TriggerTypeManual = "manual" - TriggerTypeScheduled = "scheduled" - TriggerTypeEventBased = "event_based" + TriggerTypeManual TriggerType = "manual" + TriggerTypeScheduled TriggerType = "scheduled" + TriggerTypeEventBased TriggerType = "event_based" ) // Policy defines the structure of a replication policy diff --git a/src/replication/ng/operation/controller.go b/src/replication/ng/operation/controller.go index a1c7151f9..6ac2a0b57 100644 --- a/src/replication/ng/operation/controller.go +++ b/src/replication/ng/operation/controller.go @@ -33,7 +33,7 @@ import ( // stop, query, etc. type Controller interface { // trigger is used to specified that what this replication is triggered by - StartReplication(policy *model.Policy, resource *model.Resource, trigger string) (int64, error) + StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) StopReplication(int64) error ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error) GetExecution(int64) (*models.Execution, error) @@ -58,7 +58,7 @@ type controller struct { scheduler scheduler.Scheduler } -func (c *controller) StartReplication(policy *model.Policy, resource *model.Resource, trigger string) (int64, error) { +func (c *controller) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) { if resource != nil && len(resource.Metadata.Vtags) != 1 { return 0, fmt.Errorf("the length of Vtags must be 1: %v", resource.Metadata.Vtags) } @@ -186,7 +186,7 @@ func (c *controller) GetTaskLog(taskID int64) ([]byte, error) { } // create the execution record in database -func createExecution(mgr execution.Manager, policyID int64, trigger string) (int64, error) { +func createExecution(mgr execution.Manager, policyID int64, trigger model.TriggerType) (int64, error) { id, err := mgr.Create(&models.Execution{ PolicyID: policyID, Trigger: trigger, diff --git a/src/replication/ng/operation/hook/task_test.go b/src/replication/ng/operation/hook/task_test.go index 6a88a390b..6e2de4b15 100644 --- a/src/replication/ng/operation/hook/task_test.go +++ b/src/replication/ng/operation/hook/task_test.go @@ -28,7 +28,7 @@ type fakedOperationController struct { status string } -func (f *fakedOperationController) StartReplication(*model.Policy, *model.Resource, string) (int64, error) { +func (f *fakedOperationController) StartReplication(*model.Policy, *model.Resource, model.TriggerType) (int64, error) { return 0, nil } func (f *fakedOperationController) StopReplication(int64) error { diff --git a/src/replication/ng/policy/manager/manager.go b/src/replication/ng/policy/manager/manager.go index 980857832..7fbb78303 100644 --- a/src/replication/ng/policy/manager/manager.go +++ b/src/replication/ng/policy/manager/manager.go @@ -17,9 +17,11 @@ package manager import ( "encoding/json" "errors" + "fmt" "strings" "time" + "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/replication/ng/dao" persist_models "github.com/goharbor/harbor/src/replication/ng/dao/models" "github.com/goharbor/harbor/src/replication/ng/model" @@ -61,34 +63,19 @@ func convertFromPersistModel(policy *persist_models.RepPolicy) (*model.Policy, e ply.SrcNamespaces = strings.Split(policy.SrcNamespaces, ",") } - // TODO need to consider the consistence with the policies from previous versions - // of Harbor - // both for filter and trigger - // 2. parse Filters - if len(policy.Filters) > 0 { - filters := []*model.Filter{} - if err := json.Unmarshal([]byte(policy.Filters), &filters); err != nil { - return nil, err - } - // convert the type of value from string to model.ResourceType if the filter - // is a resource type filter - for _, filter := range filters { - if filter.Type == model.FilterTypeResource { - filter.Value = (model.ResourceType)(filter.Value.(string)) - } - } - ply.Filters = filters + filters, err := parseFilters(policy.Filters) + if err != nil { + return nil, err } + ply.Filters = filters // 3. parse Trigger - if len(policy.Trigger) > 0 { - trigger := &model.Trigger{} - if err := json.Unmarshal([]byte(policy.Trigger), trigger); err != nil { - return nil, err - } - ply.Trigger = trigger + trigger, err := parseTrigger(policy.Trigger) + if err != nil { + return nil, err } + ply.Trigger = trigger return &ply, nil } @@ -216,3 +203,114 @@ func (m *DefaultManager) Update(policy *model.Policy, props ...string) error { func (m *DefaultManager) Remove(policyID int64) error { return dao.DeleteRepPolicy(policyID) } + +type filter struct { + Type model.FilterType `json:"type"` + Value interface{} `json:"value"` + Kind string `json:"kind"` + Pattern string `json:"pattern"` +} + +type trigger struct { + Type model.TriggerType `json:"type"` + Settings *model.TriggerSettings `json:"trigger_settings"` + Kind string `json:"kind"` + ScheduleParam *scheduleParam `json:"schedule_param"` +} + +type scheduleParam struct { + Type string `json:"type"` + Weekday int8 `json:"weekday"` + Offtime int64 `json:"offtime"` +} + +func parseFilters(str string) ([]*model.Filter, error) { + if len(str) == 0 { + return nil, nil + } + items := []*filter{} + if err := json.Unmarshal([]byte(str), &items); err != nil { + return nil, err + } + + filters := []*model.Filter{} + for _, item := range items { + filter := &model.Filter{ + Type: item.Type, + Value: item.Value, + } + // keep backwards compatibility + if len(filter.Type) == 0 { + switch item.Kind { + case "repository": + filter.Type = model.FilterTypeName + case "tag": + filter.Type = model.FilterTypeTag + case "label": + // TODO if we support the label filter, remove the checking logic here + continue + default: + log.Warningf("unknown filter type: %s", filter.Type) + continue + } + } + if filter.Value == nil { + filter.Value = item.Pattern + } + // convert the type of value from string to model.ResourceType if the filter + // is a resource type filter + if filter.Type == model.FilterTypeResource { + filter.Value = (model.ResourceType)(filter.Value.(string)) + } + filters = append(filters, filter) + } + return filters, nil +} + +func parseTrigger(str string) (*model.Trigger, error) { + if len(str) == 0 { + return nil, nil + } + item := &trigger{} + if err := json.Unmarshal([]byte(str), item); err != nil { + return nil, err + } + trigger := &model.Trigger{ + Type: item.Type, + Settings: item.Settings, + } + // keep backwards compatibility + if len(trigger.Type) == 0 { + switch item.Kind { + case "Manual": + trigger.Type = model.TriggerTypeManual + case "Immediate": + trigger.Type = model.TriggerTypeEventBased + case "Scheduled": + trigger.Type = model.TriggerTypeScheduled + trigger.Settings = &model.TriggerSettings{ + Cron: parseScheduleParamToCron(item.ScheduleParam), + } + default: + log.Warningf("unknown trigger type: %s", item.Kind) + return nil, nil + } + } + return trigger, nil +} + +func parseScheduleParamToCron(param *scheduleParam) string { + if param == nil { + return "" + } + offtime := param.Offtime + offtime = offtime % (3600 * 24) + hour := int(offtime / 3600) + offtime = offtime % 3600 + minute := int(offtime / 60) + second := int(offtime % 60) + if param.Type == "Weekly" { + return fmt.Sprintf("%d %d %d * * %d", second, minute, hour, param.Weekday%7) + } + return fmt.Sprintf("%d %d %d * * *", second, minute, hour) +} diff --git a/src/replication/ng/policy/manager/manager_test.go b/src/replication/ng/policy/manager/manager_test.go index 18e56a667..1ca3256b7 100644 --- a/src/replication/ng/policy/manager/manager_test.go +++ b/src/replication/ng/policy/manager/manager_test.go @@ -203,3 +203,70 @@ func TestNewDefaultManager(t *testing.T) { }) } } + +func TestParseFilters(t *testing.T) { + // nil filter string + str := "" + filters, err := parseFilters(str) + require.Nil(t, err) + assert.Nil(t, filters) + // only contains the fields that introduced in the latest version + str = `[{"type":"name","value":"library/hello-world"}]` + filters, err = parseFilters(str) + require.Nil(t, err) + require.Equal(t, 1, len(filters)) + assert.Equal(t, model.FilterTypeName, filters[0].Type) + assert.Equal(t, "library/hello-world", filters[0].Value.(string)) + // contains "kind" from previous versions + str = `[{"kind":"repository","value":"library/hello-world"}]` + filters, err = parseFilters(str) + require.Nil(t, err) + require.Equal(t, 1, len(filters)) + assert.Equal(t, model.FilterTypeName, filters[0].Type) + assert.Equal(t, "library/hello-world", filters[0].Value.(string)) + // contains "pattern" from previous versions + str = `[{"kind":"repository","pattern":"library/hello-world"}]` + filters, err = parseFilters(str) + require.Nil(t, err) + require.Equal(t, 1, len(filters)) + assert.Equal(t, model.FilterTypeName, filters[0].Type) + assert.Equal(t, "library/hello-world", filters[0].Value.(string)) +} + +func TestParseTrigger(t *testing.T) { + // nil trigger string + str := "" + trigger, err := parseTrigger(str) + require.Nil(t, err) + assert.Nil(t, trigger) + // only contains the fields that introduced in the latest version + str = `{"type":"scheduled", "trigger_settings":{"cron":"1 * * * * *"}}` + trigger, err = parseTrigger(str) + require.Nil(t, err) + assert.Equal(t, model.TriggerTypeScheduled, trigger.Type) + assert.Equal(t, "1 * * * * *", trigger.Settings.Cron) + // contains "kind" from previous versions + str = `{"kind":"Manual"}` + trigger, err = parseTrigger(str) + require.Nil(t, err) + assert.Equal(t, model.TriggerTypeManual, trigger.Type) + assert.Nil(t, trigger.Settings) + // contains "kind" from previous versions + str = `{"kind":"Immediate"}` + trigger, err = parseTrigger(str) + require.Nil(t, err) + assert.Equal(t, model.TriggerTypeEventBased, trigger.Type) + assert.Nil(t, trigger.Settings) + // contains "schedule_param" from previous versions + str = `{"kind":"Scheduled","schedule_param":{"type":"Weekly","weekday":1,"offtime":0}}` + trigger, err = parseTrigger(str) + require.Nil(t, err) + assert.Equal(t, model.TriggerTypeScheduled, trigger.Type) + assert.Equal(t, "0 0 0 * * 1", trigger.Settings.Cron) + // contains "schedule_param" from previous versions + str = `{"kind":"Scheduled","schedule_param":{"type":"Daily","offtime":0}}` + trigger, err = parseTrigger(str) + require.Nil(t, err) + assert.Equal(t, model.TriggerTypeScheduled, trigger.Type) + assert.Equal(t, "0 0 0 * * *", trigger.Settings.Cron) +}