Merge pull request #3534 from ywk253100/171101_poilicy

Update replication policy API to support trigger and filter

docs/swagger.yaml

@@ -2397,9 +2397,22 @@ definitions:
       description:
         type: string
         description: The description of the policy.
-      cron_str:
+      trigger:
+        type: object
+        description: The trigger for schedule job.
+        items:
+          $ref: '#/definitions/RepTrigger'
+      filters:
+        type: array
+        description: The replication policy filter array.
+        items:
+          $ref: '#/definitions/RepFilter'
+      replicate_existing_image_now:
         type: string
-        description: The cron string for schedule job.
+        description: Whether to replicate the existing images now.
+      replicate_deletion:
+        type: string
+        description: Whether to replicate the deletion operation.
       start_time:
         type: string
         description: The start time of the policy.
@@ -2428,6 +2441,22 @@ definitions:
       name:
         type: string
         description: The policy name.
+      trigger:
+        type: object
+        description: The trigger for schedule job.
+        items:
+          $ref: '#/definitions/RepTrigger'
+      filters:
+        type: array
+        description: The replication policy filter array.
+        items:
+          $ref: '#/definitions/RepFilter'
+      replicate_existing_image_now:
+        type: string
+        description: Whether to replicate the existing images now.
+      replicate_deletion:
+        type: string
+        description: Whether replication deletion operation.
       enabled:
         type: integer
         format: int
@@ -2449,9 +2478,40 @@ definitions:
       description:
         type: string
         description: The description of the policy.
-      cron_str:
+      trigger:
+        type: object
+        description: The trigger for schedule job.
+        items:
+          $ref: '#/definitions/RepTrigger'
+      filters:
+        type: array
+        description: The replication policy filter array.
+        items:
+          $ref: '#/definitions/RepFilter'
+      replicate_existing_image_now:
         type: string
-        description: The cron string for schedule job.
+        description: Whether to replicate the existing images now.
+      replicate_deletion:
+        type: string
+        description: Whether replication deletion operation.
+  RepTrigger:
+    type: object
+    properties:
+      type:
+        type: string
+        description: The replication policy trigger type.
+      params:
+        type: object
+        description: The map is the replication policy trigger parameters.
+  RepFilter:
+    type: object
+    properties:
+      type:
+        type: string
+        description: The replication policy filter type.
+      value:
+        type: string
+        description: The replication policy filter value.
   RepPolicyEnablementReq:
     type: object
     properties:

make/common/db/registry.sql

@@ -145,6 +145,8 @@ create table replication_policy (
  description text,
  deleted tinyint (1) DEFAULT 0 NOT NULL,
  cron_str varchar(256),
+ filters varchar(1024),
+ replicate_deletion tinyint (1) DEFAULT 0 NOT NULL,
  start_time timestamp NULL,
  creation_time timestamp default CURRENT_TIMESTAMP,
  update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,

make/common/db/registry_sqlite.sql

@@ -141,6 +141,8 @@ create table replication_policy (
  description text,
  deleted tinyint (1) DEFAULT 0 NOT NULL,
  cron_str varchar(256),
+ filters varchar(1024),
+ replicate_deletion tinyint (1) DEFAULT 0 NOT NULL,
  start_time timestamp NULL,
  creation_time timestamp default CURRENT_TIMESTAMP,
  update_time timestamp default CURRENT_TIMESTAMP

src/common/dao/replication_job.go

@@ -104,29 +104,29 @@ func FilterRepTargets(name string) ([]*models.RepTarget, error) {
 
 // AddRepPolicy ...
 func AddRepPolicy(policy models.RepPolicy) (int64, error) {
-	o := GetOrmer()
-	sql := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, start_time, creation_time, update_time ) values (?, ?, ?, ?, ?, ?, ?, ?, ?)`
-	p, err := o.Raw(sql).Prepare()
-	if err != nil {
+	if err := policy.Marshal(); err != nil {
 		return 0, err
 	}
 
+	o := GetOrmer()
+	sql := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, start_time, creation_time, update_time, filters, replicate_deletion) 
+				values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
 	params := []interface{}{}
-	params = append(params, policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.CronStr)
+	params = append(params, policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.TriggerInDB)
 	now := time.Now()
 	if policy.Enabled == 1 {
 		params = append(params, now)
 	} else {
 		params = append(params, nil)
 	}
-	params = append(params, now, now)
+	params = append(params, now, now, policy.FiltersInDB, policy.ReplicateDeletion)
 
-	r, err := p.Exec(params...)
+	result, err := o.Raw(sql, params...).Exec()
 	if err != nil {
 		return 0, err
 	}
-	id, err := r.LastInsertId()
-	return id, err
+
+	return result.LastInsertId()
 }
 
 // GetRepPolicy ...
@@ -143,6 +143,10 @@ func GetRepPolicy(id int64) (*models.RepPolicy, error) {
 		return nil, err
 	}
 
+	if err := policy.Unmarshal(); err != nil {
+		return nil, err
+	}
+
 	return &policy, nil
 }
 
@@ -154,7 +158,8 @@ func FilterRepPolicies(name string, projectID int64) ([]*models.RepPolicy, error
 
 	sql := `select rp.id, rp.project_id, rp.target_id, 
 				rt.name as target_name, rp.name, rp.enabled, rp.description,
-				rp.cron_str, rp.start_time, rp.creation_time, rp.update_time, 
+				rp.cron_str, rp.filters, rp.replicate_deletion,rp.start_time, 
+				rp.creation_time, rp.update_time, 
 				count(rj.status) as error_job_count 
 			from replication_policy rp 
 			left join replication_target rt on rp.target_id=rt.id 
@@ -180,6 +185,13 @@ func FilterRepPolicies(name string, projectID int64) ([]*models.RepPolicy, error
 	if _, err := o.Raw(sql, args).QueryRows(&policies); err != nil {
 		return nil, err
 	}
+
+	for _, policy := range policies {
+		if err := policy.Unmarshal(); err != nil {
+			return nil, err
+		}
+	}
+
 	return policies, nil
 }
 
@@ -197,6 +209,10 @@ func GetRepPolicyByName(name string) (*models.RepPolicy, error) {
 		return nil, err
 	}
 
+	if err := policy.Unmarshal(); err != nil {
+		return nil, err
+	}
+
 	return &policy, nil
 }
 
@@ -211,6 +227,12 @@ func GetRepPolicyByProject(projectID int64) ([]*models.RepPolicy, error) {
 		return nil, err
 	}
 
+	for _, policy := range policies {
+		if err := policy.Unmarshal(); err != nil {
+			return nil, err
+		}
+	}
+
 	return policies, nil
 }
 
@@ -225,6 +247,12 @@ func GetRepPolicyByTarget(targetID int64) ([]*models.RepPolicy, error) {
 		return nil, err
 	}
 
+	for _, policy := range policies {
+		if err := policy.Unmarshal(); err != nil {
+			return nil, err
+		}
+	}
+
 	return policies, nil
 }
 
@@ -239,14 +267,24 @@ func GetRepPolicyByProjectAndTarget(projectID, targetID int64) ([]*models.RepPol
 		return nil, err
 	}
 
+	for _, policy := range policies {
+		if err := policy.Unmarshal(); err != nil {
+			return nil, err
+		}
+	}
+
 	return policies, nil
 }
 
 // UpdateRepPolicy ...
 func UpdateRepPolicy(policy *models.RepPolicy) error {
+	if err := policy.Marshal(); err != nil {
+		return err
+	}
 	o := GetOrmer()
 	policy.UpdateTime = time.Now()
-	_, err := o.Update(policy, "TargetID", "Name", "Enabled", "Description", "CronStr", "UpdateTime")
+	_, err := o.Update(policy, "TargetID", "Name", "Enabled", "Description",
+		"TriggerInDB", "FiltersInDB", "ReplicateDeletion", "UpdateTime")
 	return err
 }
 

src/common/models/replication_job.go

@@ -15,10 +15,13 @@
 package models
 
 import (
+	"encoding/json"
+	"fmt"
 	"time"
 
 	"github.com/astaxie/beego/validation"
 	"github.com/vmware/harbor/src/common/utils"
+	"github.com/vmware/harbor/src/replication"
 )
 
 const (
@@ -38,21 +41,25 @@ const (
 
 // RepPolicy is the model for a replication policy, which associate to a project and a target (destination)
 type RepPolicy struct {
-	ID          int64  `orm:"pk;auto;column(id)" json:"id"`
-	ProjectID   int64  `orm:"column(project_id)" json:"project_id"`
-	ProjectName string `json:"project_name,omitempty"`
-	TargetID    int64  `orm:"column(target_id)" json:"target_id"`
-	TargetName  string `json:"target_name,omitempty"`
-	Name        string `orm:"column(name)" json:"name"`
-	//	Target       RepTarget `orm:"-" json:"target"`
-	Enabled       int       `orm:"column(enabled)" json:"enabled"`
-	Description   string    `orm:"column(description)" json:"description"`
-	CronStr       string    `orm:"column(cron_str)" json:"cron_str"`
-	StartTime     time.Time `orm:"column(start_time)" json:"start_time"`
-	CreationTime  time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
-	UpdateTime    time.Time `orm:"column(update_time);auto_now" json:"update_time"`
-	ErrorJobCount int       `json:"error_job_count"`
-	Deleted       int       `orm:"column(deleted)" json:"deleted"`
+	ID                        int64        `orm:"pk;auto;column(id)" json:"id"`
+	ProjectID                 int64        `orm:"column(project_id)" json:"project_id"`
+	ProjectName               string       `orm:"-" json:"project_name,omitempty"`
+	TargetID                  int64        `orm:"column(target_id)" json:"target_id"`
+	TargetName                string       `json:"target_name,omitempty"`
+	Name                      string       `orm:"column(name)" json:"name"`
+	Enabled                   int          `orm:"column(enabled)" json:"enabled"`
+	Description               string       `orm:"column(description)" json:"description"`
+	Trigger                   *RepTrigger  `orm:"-" json:"trigger"`
+	TriggerInDB               string       `orm:"column(cron_str)" json:"-"`
+	Filters                   []*RepFilter `orm:"-" json:"filters"`
+	FiltersInDB               string       `orm:"column(filters)" json:"-"`
+	ReplicateExistingImageNow bool         `orm:"-" json:"replicate_existing_image_now"`
+	ReplicateDeletion         bool         `orm:"column(replicate_deletion)" json:"replicate_deletion"`
+	StartTime                 time.Time    `orm:"column(start_time)" json:"start_time"`
+	CreationTime              time.Time    `orm:"column(creation_time);auto_now_add" json:"creation_time"`
+	UpdateTime                time.Time    `orm:"column(update_time);auto_now" json:"update_time"`
+	ErrorJobCount             int          `orm:"-" json:"error_job_count"`
+	Deleted                   int          `orm:"column(deleted)" json:"deleted"`
 }
 
 // Valid ...
@@ -77,8 +84,98 @@ func (r *RepPolicy) Valid(v *validation.Validation) {
 		v.SetError("enabled", "must be 0 or 1")
 	}
 
-	if len(r.CronStr) > 256 {
-		v.SetError("cron_str", "max length is 256")
+	if r.Trigger != nil {
+		r.Trigger.Valid(v)
+	}
+
+	for _, filter := range r.Filters {
+		filter.Valid(v)
+	}
+
+	if err := r.Marshal(); err != nil {
+		v.SetError("trigger or filters", err.Error())
+	}
+
+	if len(r.TriggerInDB) > 256 {
+		v.SetError("trigger", "max length is 256")
+	}
+
+	if len(r.FiltersInDB) > 1024 {
+		v.SetError("filters", "max length is 1024")
+	}
+}
+
+// Marshal marshal RepTrigger and RepFilter array to json string
+func (r *RepPolicy) Marshal() error {
+	if r.Trigger != nil {
+		b, err := json.Marshal(r.Trigger)
+		if err != nil {
+			return err
+		}
+		r.TriggerInDB = string(b)
+	}
+
+	if r.Filters != nil {
+		b, err := json.Marshal(r.Filters)
+		if err != nil {
+			return err
+		}
+		r.FiltersInDB = string(b)
+	}
+	return nil
+}
+
+// Unmarshal unmarshal json string to RepTrigger and RepFilter array
+func (r *RepPolicy) Unmarshal() error {
+	if len(r.TriggerInDB) > 0 {
+		trigger := &RepTrigger{}
+		if err := json.Unmarshal([]byte(r.TriggerInDB), &trigger); err != nil {
+			return err
+		}
+		r.Trigger = trigger
+	}
+
+	if len(r.FiltersInDB) > 0 {
+		filter := []*RepFilter{}
+		if err := json.Unmarshal([]byte(r.FiltersInDB), &filter); err != nil {
+			return err
+		}
+		r.Filters = filter
+	}
+	return nil
+}
+
+// RepFilter holds information for the replication policy filter
+type RepFilter struct {
+	Type  string `json:"type"`
+	Value string `json:"value"`
+}
+
+// Valid ...
+func (r *RepFilter) Valid(v *validation.Validation) {
+	if !(r.Type == replication.FilterItemKindProject ||
+		r.Type == replication.FilterItemKindRepository ||
+		r.Type == replication.FilterItemKindTag) {
+		v.SetError("filter.type", fmt.Sprintf("invalid filter type: %s", r.Type))
+	}
+
+	if len(r.Value) == 0 {
+		v.SetError("filter.value", "can not be empty")
+	}
+}
+
+// RepTrigger holds information for the replication policy trigger
+type RepTrigger struct {
+	Type   string                 `json:"type"`
+	Params map[string]interface{} `json:"params"`
+}
+
+// Valid ...
+func (r *RepTrigger) Valid(v *validation.Validation) {
+	if !(r.Type == replication.TriggerKindManually ||
+		r.Type == replication.TriggerKindSchedule ||
+		r.Type == replication.TriggerKindImmediately) {
+		v.SetError("trigger.type", fmt.Sprintf("invalid trigger type: %s", r.Type))
 	}
 }
 

src/common/models/replication_test.go

@@ -0,0 +1,50 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMarshalAndUnmarshal(t *testing.T) {
+	trigger := &RepTrigger{
+		Type:   "schedule",
+		Params: map[string]interface{}{"date": "2:00"},
+	}
+	filters := []*RepFilter{
+		&RepFilter{
+			Type:  "repository",
+			Value: "library/ubuntu*",
+		},
+	}
+	policy := &RepPolicy{
+		Trigger: trigger,
+		Filters: filters,
+	}
+
+	err := policy.Marshal()
+	require.Nil(t, err)
+
+	policy.Trigger = nil
+	policy.Filters = nil
+	err = policy.Unmarshal()
+	require.Nil(t, err)
+
+	assert.EqualValues(t, filters, policy.Filters)
+	assert.EqualValues(t, trigger, policy.Trigger)
+}

src/replication/consts.go

@@ -8,6 +8,13 @@ const (
 	//FilterItemKindTag : Kind of filter item is 'tag'
 	FilterItemKindTag = "tag"
 
+	//TriggerKindManually : kind of trigger is 'manully'
+	TriggerKindManually = "manually"
+	//TriggerKindSchedule : kind of trigger is 'schedule'
+	TriggerKindSchedule = "schedule"
+	//TriggerKindImmediately : kind of trigger is 'immediately'
+	TriggerKindImmediately = "immediately"
+
 	//AdaptorKindHarbor : Kind of adaptor of Harbor
 	AdaptorKindHarbor = "Harbor"
 )

src/ui/api/harborapi_test.go

@@ -711,7 +711,10 @@ func (a testapi) AddPolicy(authInfo usrInfo, repPolicy apilib.RepPolicyPost) (in
 	_sling = _sling.Path(path)
 	_sling = _sling.BodyJSON(repPolicy)
 
-	httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo)
+	httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo)
+	if httpStatusCode != http.StatusCreated {
+		log.Println(string(body))
+	}
 	return httpStatusCode, err
 }
 

src/ui/api/replication_policy_test.go

@@ -15,10 +15,14 @@ package api
 
 import (
 	"fmt"
-	"github.com/stretchr/testify/assert"
-	"github.com/vmware/harbor/tests/apitests/apilib"
 	"strconv"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/replication"
+	"github.com/vmware/harbor/tests/apitests/apilib"
 )
 
 const (
@@ -37,7 +41,19 @@ func TestPoliciesPost(t *testing.T) {
 	//add target
 	CommonAddTarget()
 	targetID := int64(CommonGetTarget())
-	repPolicy := &apilib.RepPolicyPost{int64(1), targetID, addPolicyName}
+	repPolicy := &apilib.RepPolicyPost{int64(1), targetID, addPolicyName,
+		&models.RepTrigger{
+			Type: replication.TriggerKindSchedule,
+			Params: map[string]interface{}{
+				"date": "2:00",
+			},
+		},
+		[]*models.RepFilter{
+			&models.RepFilter{
+				Type:  replication.FilterItemKindRepository,
+				Value: "library/ubuntu*",
+			},
+		}}
 
 	fmt.Println("Testing Policies Post API")
 
@@ -52,7 +68,7 @@ func TestPoliciesPost(t *testing.T) {
 	}
 
 	//-------------------case 2 : response code = 409------------------------//
-	fmt.Println("case 1 : response code = 409:policy already exists")
+	fmt.Println("case 2 : response code = 409:policy already exists")
 	httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy)
 	if err != nil {
 		t.Error("Error while add policy", err.Error())
@@ -108,7 +124,7 @@ func TestPoliciesPost(t *testing.T) {
 	}
 
 	//-------------------case 7 : response code = 400------------------------//
-	fmt.Println("case 6 : response code = 400:target_id does not exist.")
+	fmt.Println("case 7 : response code = 400:target_id does not exist.")
 
 	repPolicy.TargetId = int64(1111)
 	httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy)
@@ -119,6 +135,20 @@ func TestPoliciesPost(t *testing.T) {
 		assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400")
 	}
 
+	fmt.Println("case 8 : response code = 400: invalid filter")
+	repPolicy = &apilib.RepPolicyPost{int64(1), targetID, addPolicyName,
+		&models.RepTrigger{
+			Type: replication.TriggerKindManually,
+		},
+		[]*models.RepFilter{
+			&models.RepFilter{
+				Type:  "replication",
+				Value: "",
+			},
+		}}
+	httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy)
+	require.Nil(t, err)
+	assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400")
 }
 
 func TestPoliciesList(t *testing.T) {

tests/apitests/apilib/rep_policy_post.go

@@ -1,10 +1,10 @@
-/* 
+/*
  * Harbor API
  *
  * These APIs provide services for manipulating Harbor project.
  *
  * OpenAPI spec version: 0.3.0
- * 
+ *
  * Generated by: https://github.com/swagger-api/swagger-codegen.git
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,6 +22,10 @@
 
 package apilib
 
+import (
+	"github.com/vmware/harbor/src/common/models"
+)
+
 type RepPolicyPost struct {
 
 	// The project ID.
@@ -32,4 +36,10 @@ type RepPolicyPost struct {
 
 	// The policy name.
 	Name string `json:"name,omitempty"`
+
+	// Trigger
+	Trigger *models.RepTrigger `json:"trigger"`
+
+	// Filters
+	Filters []*models.RepFilter `json:"filters"`
 }

tools/migration/changelog.md

@@ -56,3 +56,8 @@ Changelog for harbor database schema
   - insert data into table `project_metadata`
   - delete column `public` from table `project`
   - add column `insecure` to table `replication_target`
+
+## 1.3.1
+
+  - add column `filters` to table `replication_policy`
+  - add column `replicate_deletion` to table `replication_policy`