Update replication policy API to support trigger and filter

docs/swagger.yaml

@@ -2397,9 +2397,18 @@ definitions:
       description:
         type: string
         description: The description of the policy.
-      cron_str:
+      trigger:
         type: string
-        description: The cron string for schedule job.
+        description: The trigger for schedule job.
+      filters:
+        type: array
+        description: >-
+          The replication policy filter array.
+        items:
+          $ref: '#/definitions/RepFilter'
+      replicate_deletion:
+        type: string
+        description: Whether replication deletion operation.
       start_time:
         type: string
         description: The start time of the policy.
@@ -2428,6 +2437,18 @@ definitions:
       name:
         type: string
         description: The policy name.
+      trigger:
+        type: string
+        description: The trigger for schedule job.
+      filters:
+        type: array
+        description: >-
+          The replication policy filter array.
+        items:
+          $ref: '#/definitions/RepFilter'
+      replicate_deletion:
+        type: string
+        description: Whether replication deletion operation.
       enabled:
         type: integer
         format: int
@@ -2449,9 +2470,26 @@ definitions:
       description:
         type: string
         description: The description of the policy.
-      cron_str:
+      trigger:
         type: string
         description: The cron string for schedule job.
+      filters:
+        type: array
+        description: The replication policy filter array.
+        items:
+          $ref: '#/definitions/RepFilter'
+      replicate_deletion:
+        type: string
+        description: Whether replication deletion operation.
+  RepFilter:
+    type: object
+    properties:
+      type:
+        type: string
+        description: The replication policy filter type.
+      value:
+        type: string
+        description: The replication policy filter value.
   RepPolicyEnablementReq:
     type: object
     properties:

make/common/db/registry.sql

@@ -145,6 +145,8 @@ create table replication_policy (
  description text,
  deleted tinyint (1) DEFAULT 0 NOT NULL,
  cron_str varchar(256),
+ filters varchar(1024),
+ replicate_deletion tinyint (1) DEFAULT 0 NOT NULL,
  start_time timestamp NULL,
  creation_time timestamp default CURRENT_TIMESTAMP,
  update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,

make/common/db/registry_sqlite.sql

@@ -141,6 +141,8 @@ create table replication_policy (
  description text,
  deleted tinyint (1) DEFAULT 0 NOT NULL,
  cron_str varchar(256),
+ filters varchar(1024),
+ replicate_deletion tinyint (1) DEFAULT 0 NOT NULL,
  start_time timestamp NULL,
  creation_time timestamp default CURRENT_TIMESTAMP,
  update_time timestamp default CURRENT_TIMESTAMP

src/common/dao/replication_job.go

@@ -104,29 +104,18 @@ func FilterRepTargets(name string) ([]*models.RepTarget, error) {
 
 // AddRepPolicy ...
 func AddRepPolicy(policy models.RepPolicy) (int64, error) {
-	o := GetOrmer()
-	sql := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, start_time, creation_time, update_time ) values (?, ?, ?, ?, ?, ?, ?, ?, ?)`
-	p, err := o.Raw(sql).Prepare()
-	if err != nil {
+	if err := policy.MarshalFilter(); err != nil {
 		return 0, err
 	}
-
-	params := []interface{}{}
-	params = append(params, policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.CronStr)
 	now := time.Now()
+	policy.CreationTime = now
+	policy.UpdateTime = now
 	if policy.Enabled == 1 {
-		params = append(params, now)
-	} else {
-		params = append(params, nil)
+		policy.StartTime = now
 	}
-	params = append(params, now, now)
+	policy.Deleted = 0
 
-	r, err := p.Exec(params...)
-	if err != nil {
-		return 0, err
-	}
-	id, err := r.LastInsertId()
-	return id, err
+	return GetOrmer().Insert(&policy)
 }
 
 // GetRepPolicy ...
@@ -143,6 +132,10 @@ func GetRepPolicy(id int64) (*models.RepPolicy, error) {
 		return nil, err
 	}
 
+	if err := policy.UnmarshalFilter(); err != nil {
+		return nil, err
+	}
+
 	return &policy, nil
 }
 
@@ -154,7 +147,8 @@ func FilterRepPolicies(name string, projectID int64) ([]*models.RepPolicy, error
 
 	sql := `select rp.id, rp.project_id, rp.target_id, 
 				rt.name as target_name, rp.name, rp.enabled, rp.description,
-				rp.cron_str, rp.start_time, rp.creation_time, rp.update_time, 
+				rp.cron_str, rp.filters, rp.replicate_deletion,rp.start_time, 
+				rp.creation_time, rp.update_time, 
 				count(rj.status) as error_job_count 
 			from replication_policy rp 
 			left join replication_target rt on rp.target_id=rt.id 
@@ -180,6 +174,13 @@ func FilterRepPolicies(name string, projectID int64) ([]*models.RepPolicy, error
 	if _, err := o.Raw(sql, args).QueryRows(&policies); err != nil {
 		return nil, err
 	}
+
+	for _, policy := range policies {
+		if err := policy.UnmarshalFilter(); err != nil {
+			return nil, err
+		}
+	}
+
 	return policies, nil
 }
 
@@ -197,6 +198,10 @@ func GetRepPolicyByName(name string) (*models.RepPolicy, error) {
 		return nil, err
 	}
 
+	if err := policy.UnmarshalFilter(); err != nil {
+		return nil, err
+	}
+
 	return &policy, nil
 }
 
@@ -211,6 +216,12 @@ func GetRepPolicyByProject(projectID int64) ([]*models.RepPolicy, error) {
 		return nil, err
 	}
 
+	for _, policy := range policies {
+		if err := policy.UnmarshalFilter(); err != nil {
+			return nil, err
+		}
+	}
+
 	return policies, nil
 }
 
@@ -225,6 +236,12 @@ func GetRepPolicyByTarget(targetID int64) ([]*models.RepPolicy, error) {
 		return nil, err
 	}
 
+	for _, policy := range policies {
+		if err := policy.UnmarshalFilter(); err != nil {
+			return nil, err
+		}
+	}
+
 	return policies, nil
 }
 
@@ -239,14 +256,24 @@ func GetRepPolicyByProjectAndTarget(projectID, targetID int64) ([]*models.RepPol
 		return nil, err
 	}
 
+	for _, policy := range policies {
+		if err := policy.UnmarshalFilter(); err != nil {
+			return nil, err
+		}
+	}
+
 	return policies, nil
 }
 
 // UpdateRepPolicy ...
 func UpdateRepPolicy(policy *models.RepPolicy) error {
+	if err := policy.MarshalFilter(); err != nil {
+		return err
+	}
 	o := GetOrmer()
 	policy.UpdateTime = time.Now()
-	_, err := o.Update(policy, "TargetID", "Name", "Enabled", "Description", "CronStr", "UpdateTime")
+	_, err := o.Update(policy, "TargetID", "Name", "Enabled", "Description",
+		"Trigger", "FiltersInDB", "ReplicateDeletion", "UpdateTime")
 	return err
 }
 

src/common/models/replicate_test.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMarshalAndUnmarshalFilter(t *testing.T) {
+	filters := []*RepFilter{
+		&RepFilter{
+			Type:  "repository",
+			Value: "library/ubuntu*",
+		},
+	}
+	policy := &RepPolicy{
+		Filters: filters,
+	}
+
+	err := policy.MarshalFilter()
+	require.Nil(t, err)
+
+	policy.Filters = nil
+	err = policy.UnmarshalFilter()
+	require.Nil(t, err)
+
+	assert.EqualValues(t, filters, policy.Filters)
+}

src/common/models/replication_job.go

@@ -15,10 +15,13 @@
 package models
 
 import (
+	"encoding/json"
+	"fmt"
 	"time"
 
 	"github.com/astaxie/beego/validation"
 	"github.com/vmware/harbor/src/common/utils"
+	"github.com/vmware/harbor/src/replication"
 )
 
 const (
@@ -38,21 +41,23 @@ const (
 
 // RepPolicy is the model for a replication policy, which associate to a project and a target (destination)
 type RepPolicy struct {
-	ID          int64  `orm:"pk;auto;column(id)" json:"id"`
-	ProjectID   int64  `orm:"column(project_id)" json:"project_id"`
-	ProjectName string `json:"project_name,omitempty"`
-	TargetID    int64  `orm:"column(target_id)" json:"target_id"`
-	TargetName  string `json:"target_name,omitempty"`
-	Name        string `orm:"column(name)" json:"name"`
-	//	Target       RepTarget `orm:"-" json:"target"`
-	Enabled       int       `orm:"column(enabled)" json:"enabled"`
-	Description   string    `orm:"column(description)" json:"description"`
-	CronStr       string    `orm:"column(cron_str)" json:"cron_str"`
-	StartTime     time.Time `orm:"column(start_time)" json:"start_time"`
-	CreationTime  time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
-	UpdateTime    time.Time `orm:"column(update_time);auto_now" json:"update_time"`
-	ErrorJobCount int       `json:"error_job_count"`
-	Deleted       int       `orm:"column(deleted)" json:"deleted"`
+	ID                int64        `orm:"pk;auto;column(id)" json:"id"`
+	ProjectID         int64        `orm:"column(project_id)" json:"project_id"`
+	ProjectName       string       `orm:"-" json:"project_name,omitempty"`
+	TargetID          int64        `orm:"column(target_id)" json:"target_id"`
+	TargetName        string       `orm:"-" json:"target_name,omitempty"`
+	Name              string       `orm:"column(name)" json:"name"`
+	Enabled           int          `orm:"column(enabled)" json:"enabled"`
+	Description       string       `orm:"column(description)" json:"description"`
+	Trigger           string       `orm:"column(cron_str)" json:"trigger"`
+	Filters           []*RepFilter `orm:"-" json:"filters"`
+	FiltersInDB       string       `orm:"column(filters)" json:"-"`
+	ReplicateDeletion bool         `orm:"column(replicate_deletion)" json:"replicate_deletion"`
+	StartTime         time.Time    `orm:"column(start_time)" json:"start_time"`
+	CreationTime      time.Time    `orm:"column(creation_time);auto_now_add" json:"creation_time"`
+	UpdateTime        time.Time    `orm:"column(update_time);auto_now" json:"update_time"`
+	ErrorJobCount     int          `orm:"-" json:"error_job_count"`
+	Deleted           int          `orm:"column(deleted)" json:"deleted"`
 }
 
 // Valid ...
@@ -77,8 +82,62 @@ func (r *RepPolicy) Valid(v *validation.Validation) {
 		v.SetError("enabled", "must be 0 or 1")
 	}
 
-	if len(r.CronStr) > 256 {
-		v.SetError("cron_str", "max length is 256")
+	if len(r.Trigger) > 256 {
+		v.SetError("trigger", "max length is 256")
+	}
+
+	for _, filter := range r.Filters {
+		filter.Valid(v)
+	}
+
+	if err := r.MarshalFilter(); err != nil {
+		v.SetError("filters", err.Error())
+	}
+	if len(r.Filters) > 1024 {
+		v.SetError("filters", "max length is 1024")
+	}
+}
+
+// MarshalFilter marshal RepFilter array to json string
+func (r *RepPolicy) MarshalFilter() error {
+	if r.Filters != nil {
+		b, err := json.Marshal(r.Filters)
+		if err != nil {
+			return err
+		}
+		r.FiltersInDB = string(b)
+	}
+	return nil
+}
+
+// UnmarshalFilter unmarshal json string to RepFilter array
+func (r *RepPolicy) UnmarshalFilter() error {
+	if len(r.FiltersInDB) > 0 {
+		filter := []*RepFilter{}
+		if err := json.Unmarshal([]byte(r.FiltersInDB), &filter); err != nil {
+			return err
+		}
+		r.Filters = filter
+	}
+	return nil
+}
+
+// RepFilter holds information for the replication policy filter
+type RepFilter struct {
+	Type  string `json:"type"`
+	Value string `json:"value"`
+}
+
+// Valid ...
+func (r *RepFilter) Valid(v *validation.Validation) {
+	if !(r.Type == replication.FilterItemKindProject ||
+		r.Type == replication.FilterItemKindRepository ||
+		r.Type == replication.FilterItemKindTag) {
+		v.SetError("filter.type", fmt.Sprintf("invalid filter type: %s", r.Type))
+	}
+
+	if len(r.Value) == 0 {
+		v.SetError("filter.value", "can not be empty")
 	}
 }
 

src/ui/api/harborapi_test.go

@@ -711,7 +711,10 @@ func (a testapi) AddPolicy(authInfo usrInfo, repPolicy apilib.RepPolicyPost) (in
 	_sling = _sling.Path(path)
 	_sling = _sling.BodyJSON(repPolicy)
 
-	httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo)
+	httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo)
+	if httpStatusCode != http.StatusCreated {
+		log.Println(string(body))
+	}
 	return httpStatusCode, err
 }
 

src/ui/api/replication_policy_test.go

@@ -15,10 +15,14 @@ package api
 
 import (
 	"fmt"
-	"github.com/stretchr/testify/assert"
-	"github.com/vmware/harbor/tests/apitests/apilib"
 	"strconv"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/replication"
+	"github.com/vmware/harbor/tests/apitests/apilib"
 )
 
 const (
@@ -37,7 +41,12 @@ func TestPoliciesPost(t *testing.T) {
 	//add target
 	CommonAddTarget()
 	targetID := int64(CommonGetTarget())
-	repPolicy := &apilib.RepPolicyPost{int64(1), targetID, addPolicyName}
+	repPolicy := &apilib.RepPolicyPost{int64(1), targetID, addPolicyName, []*models.RepFilter{
+		&models.RepFilter{
+			Type:  replication.FilterItemKindRepository,
+			Value: "library/ubuntu*",
+		},
+	}}
 
 	fmt.Println("Testing Policies Post API")
 
@@ -52,7 +61,7 @@ func TestPoliciesPost(t *testing.T) {
 	}
 
 	//-------------------case 2 : response code = 409------------------------//
-	fmt.Println("case 1 : response code = 409:policy already exists")
+	fmt.Println("case 2 : response code = 409:policy already exists")
 	httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy)
 	if err != nil {
 		t.Error("Error while add policy", err.Error())
@@ -108,7 +117,7 @@ func TestPoliciesPost(t *testing.T) {
 	}
 
 	//-------------------case 7 : response code = 400------------------------//
-	fmt.Println("case 6 : response code = 400:target_id does not exist.")
+	fmt.Println("case 7 : response code = 400:target_id does not exist.")
 
 	repPolicy.TargetId = int64(1111)
 	httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy)
@@ -119,6 +128,16 @@ func TestPoliciesPost(t *testing.T) {
 		assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400")
 	}
 
+	fmt.Println("case 8 : response code = 400: invalid filter")
+	repPolicy = &apilib.RepPolicyPost{int64(1), targetID, addPolicyName, []*models.RepFilter{
+		&models.RepFilter{
+			Type:  "replication",
+			Value: "",
+		},
+	}}
+	httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy)
+	require.Nil(t, err)
+	assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400")
 }
 
 func TestPoliciesList(t *testing.T) {

tests/apitests/apilib/rep_policy_post.go

@@ -22,6 +22,10 @@
 
 package apilib
 
+import (
+	"github.com/vmware/harbor/src/common/models"
+)
+
 type RepPolicyPost struct {
 
 	// The project ID.
@@ -32,4 +36,7 @@ type RepPolicyPost struct {
 
 	// The policy name.
 	Name string `json:"name,omitempty"`
+
+	// Filters
+	Filters []*models.RepFilter `json:"filters"`
 }

tools/migration/changelog.md

@@ -56,3 +56,8 @@ Changelog for harbor database schema
   - insert data into table `project_metadata`
   - delete column `public` from table `project`
   - add column `insecure` to table `replication_target`
+
+## 1.3.1
+
+  - add column `filters` to table `replication_policy`
+  - add column `replicate_deletion` to table `replication_policy`