Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-06-30 16:55:07 +02:00
Code Issues Projects Releases Wiki Activity

Implement replication policy manager

Browse Source
This commit is contained in:
Wenkai Yin 2017-11-15 14:41:26 +08:00
parent 1ec240d753
commit 31cf6c078e
22 changed files with 1111 additions and 829 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
src/common/dao/replication_job.go
View File

@ -104,22 +104,18 @@ func FilterRepTargets(name string) ([]*models.RepTarget, error) {
// AddRepPolicy ...
func AddRepPolicy(policy models.RepPolicy) (int64, error) {
if err := policy.Marshal(); err != nil {
return 0, err
}
o := GetOrmer()
sql := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, start_time, creation_time, update_time, filters, replicate_deletion)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
params := []interface{}{}
params = append(params, policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.TriggerInDB)
params = append(params, policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.Trigger)
now := time.Now()
if policy.Enabled == 1 {
params = append(params, now)
} else {
params = append(params, nil)
}
params = append(params, now, now, policy.FiltersInDB, policy.ReplicateDeletion)
params = append(params, now, now, policy.Filters, policy.ReplicateDeletion)
result, err := o.Raw(sql, params...).Exec()
if err != nil {
@ -132,7 +128,7 @@ func AddRepPolicy(policy models.RepPolicy) (int64, error) {
// GetRepPolicy ...
func GetRepPolicy(id int64) (*models.RepPolicy, error) {
o := GetOrmer()
sql := `select * from replication_policy where id = ?`
sql := `select * from replication_policy where id = ? and deleted = 0`
var policy models.RepPolicy
@ -143,10 +139,6 @@ func GetRepPolicy(id int64) (*models.RepPolicy, error) {
return nil, err
}
if err := policy.Unmarshal(); err != nil {
return nil, err
}
return &policy, nil
}
@ -186,12 +178,6 @@ func FilterRepPolicies(name string, projectID int64) ([]*models.RepPolicy, error
return nil, err
}
for _, policy := range policies {
if err := policy.Unmarshal(); err != nil {
return nil, err
}
}
return policies, nil
}
@ -209,10 +195,6 @@ func GetRepPolicyByName(name string) (*models.RepPolicy, error) {
return nil, err
}
if err := policy.Unmarshal(); err != nil {
return nil, err
}
return &policy, nil
}
@ -227,12 +209,6 @@ func GetRepPolicyByProject(projectID int64) ([]*models.RepPolicy, error) {
return nil, err
}
for _, policy := range policies {
if err := policy.Unmarshal(); err != nil {
return nil, err
}
}
return policies, nil
}
@ -247,12 +223,6 @@ func GetRepPolicyByTarget(targetID int64) ([]*models.RepPolicy, error) {
return nil, err
}
for _, policy := range policies {
if err := policy.Unmarshal(); err != nil {
return nil, err
}
}
return policies, nil
}
@ -267,24 +237,15 @@ func GetRepPolicyByProjectAndTarget(projectID, targetID int64) ([]*models.RepPol
return nil, err
}
for _, policy := range policies {
if err := policy.Unmarshal(); err != nil {
return nil, err
}
}
return policies, nil
}
// UpdateRepPolicy ...
func UpdateRepPolicy(policy *models.RepPolicy) error {
if err := policy.Marshal(); err != nil {
return err
}
o := GetOrmer()
policy.UpdateTime = time.Now()
_, err := o.Update(policy, "TargetID", "Name", "Enabled", "Description",
"TriggerInDB", "FiltersInDB", "ReplicateDeletion", "UpdateTime")
"Trigger", "Filters", "ReplicateDeletion", "UpdateTime")
return err
}

152
src/common/models/replication_job.go
View File

@ -15,13 +15,10 @@
package models
import (
"encoding/json"
"fmt"
"time"
"github.com/astaxie/beego/validation"
"github.com/vmware/harbor/src/common/utils"
"github.com/vmware/harbor/src/replication"
)
const (
@ -41,142 +38,19 @@ const (
// RepPolicy is the model for a replication policy, which associate to a project and a target (destination)
type RepPolicy struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
ProjectID int64 `orm:"column(project_id)" json:"project_id"`
ProjectName string `orm:"-" json:"project_name,omitempty"`
TargetID int64 `orm:"column(target_id)" json:"target_id"`
TargetName string `json:"target_name,omitempty"`
Name string `orm:"column(name)" json:"name"`
Enabled int `orm:"column(enabled)" json:"enabled"`
Description string `orm:"column(description)" json:"description"`
Trigger *RepTrigger `orm:"-" json:"trigger"`
TriggerInDB string `orm:"column(cron_str)" json:"-"`
Filters []*RepFilter `orm:"-" json:"filters"`
FiltersInDB string `orm:"column(filters)" json:"-"`
ReplicateExistingImageNow bool `orm:"-" json:"replicate_existing_image_now"`
ReplicateDeletion bool `orm:"column(replicate_deletion)" json:"replicate_deletion"`
StartTime time.Time `orm:"column(start_time)" json:"start_time"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
ErrorJobCount int `orm:"-" json:"error_job_count"`
Deleted int `orm:"column(deleted)" json:"deleted"`
}
// Valid ...
func (r *RepPolicy) Valid(v *validation.Validation) {
if len(r.Name) == 0 {
v.SetError("name", "can not be empty")
}
if len(r.Name) > 256 {
v.SetError("name", "max length is 256")
}
if r.ProjectID <= 0 {
v.SetError("project_id", "invalid")
}
if r.TargetID <= 0 {
v.SetError("target_id", "invalid")
}
if r.Enabled != 0 && r.Enabled != 1 {
v.SetError("enabled", "must be 0 or 1")
}
if r.Trigger != nil {
r.Trigger.Valid(v)
}
for _, filter := range r.Filters {
filter.Valid(v)
}
if err := r.Marshal(); err != nil {
v.SetError("trigger or filters", err.Error())
}
if len(r.TriggerInDB) > 256 {
v.SetError("trigger", "max length is 256")
}
if len(r.FiltersInDB) > 1024 {
v.SetError("filters", "max length is 1024")
}
}
// Marshal marshal RepTrigger and RepFilter array to json string
func (r *RepPolicy) Marshal() error {
if r.Trigger != nil {
b, err := json.Marshal(r.Trigger)
if err != nil {
return err
}
r.TriggerInDB = string(b)
}
if r.Filters != nil {
b, err := json.Marshal(r.Filters)
if err != nil {
return err
}
r.FiltersInDB = string(b)
}
return nil
}
// Unmarshal unmarshal json string to RepTrigger and RepFilter array
func (r *RepPolicy) Unmarshal() error {
if len(r.TriggerInDB) > 0 {
trigger := &RepTrigger{}
if err := json.Unmarshal([]byte(r.TriggerInDB), &trigger); err != nil {
return err
}
r.Trigger = trigger
}
if len(r.FiltersInDB) > 0 {
filter := []*RepFilter{}
if err := json.Unmarshal([]byte(r.FiltersInDB), &filter); err != nil {
return err
}
r.Filters = filter
}
return nil
}
// RepFilter holds information for the replication policy filter
type RepFilter struct {
Type string `json:"type"`
Value string `json:"value"`
}
// Valid ...
func (r *RepFilter) Valid(v *validation.Validation) {
if !(r.Type == replication.FilterItemKindProject ||
r.Type == replication.FilterItemKindRepository ||
r.Type == replication.FilterItemKindTag) {
v.SetError("filter.type", fmt.Sprintf("invalid filter type: %s", r.Type))
}
if len(r.Value) == 0 {
v.SetError("filter.value", "can not be empty")
}
}
// RepTrigger holds information for the replication policy trigger
type RepTrigger struct {
Type string `json:"type"`
Params map[string]interface{} `json:"params"`
}
// Valid ...
func (r *RepTrigger) Valid(v *validation.Validation) {
if !(r.Type == replication.TriggerKindManually ||
r.Type == replication.TriggerKindSchedule ||
r.Type == replication.TriggerKindImmediately) {
v.SetError("trigger.type", fmt.Sprintf("invalid trigger type: %s", r.Type))
}
ID int64 `orm:"pk;auto;column(id)"`
ProjectID int64 `orm:"column(project_id)" `
TargetID int64 `orm:"column(target_id)"`
Name string `orm:"column(name)"`
Enabled int `orm:"column(enabled)"`
Description string `orm:"column(description)"`
Trigger string `orm:"column(cron_str)"`
Filters string `orm:"column(filters)"`
ReplicateDeletion bool `orm:"column(replicate_deletion)"`
StartTime time.Time `orm:"column(start_time)"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add"`
UpdateTime time.Time `orm:"column(update_time);auto_now"`
Deleted int `orm:"column(deleted)"`
}
// RepJob is the model for a replication job, which is the execution unit on job service, currently it is used to transfer/remove

50
src/common/models/replication_test.go
View File

@ -1,50 +0,0 @@
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMarshalAndUnmarshal(t *testing.T) {
trigger := &RepTrigger{
Type: "schedule",
Params: map[string]interface{}{"date": "2:00"},
}
filters := []*RepFilter{
&RepFilter{
Type: "repository",
Value: "library/ubuntu*",
},
}
policy := &RepPolicy{
Trigger: trigger,
Filters: filters,
}
err := policy.Marshal()
require.Nil(t, err)
policy.Trigger = nil
policy.Filters = nil
err = policy.Unmarshal()
require.Nil(t, err)
assert.EqualValues(t, filters, policy.Filters)
assert.EqualValues(t, trigger, policy.Trigger)
}

34
src/replication/core/controller.go
View File

@ -67,10 +67,13 @@ func (ctl *Controller) Init() error {
TriggerName: queryName,
}
policies := ctl.policyManager.GetPolicies(query)
policies, err := ctl.policyManager.GetPolicies(query)
if err != nil {
return err
}
if policies != nil && len(policies) > 0 {
for _, policy := range policies {
if err := ctl.triggerManager.SetupTrigger(policy.ID, policy.Trigger); err != nil {
if err := ctl.triggerManager.SetupTrigger(policy.ID, *policy.Trigger); err != nil {
//TODO: Log error
fmt.Printf("Error: %s", err)
//TODO:Update the status of policy
@ -87,35 +90,38 @@ func (ctl *Controller) Init() error {
}
//CreatePolicy is used to create a new policy and enable it if necessary
func (ctl *Controller) CreatePolicy(newPolicy models.ReplicationPolicy) error {
func (ctl *Controller) CreatePolicy(newPolicy models.ReplicationPolicy) (int64, error) {
//Validate policy
//TODO:
return nil
// TODO
return ctl.policyManager.CreatePolicy(newPolicy)
}
//UpdatePolicy will update the policy with new content.
//Parameter updatedPolicy must have the ID of the updated policy.
func (ctl *Controller) UpdatePolicy(updatedPolicy models.ReplicationPolicy) error {
return nil
// TODO check pre-conditions
return ctl.policyManager.UpdatePolicy(updatedPolicy)
}
//RemovePolicy will remove the specified policy and clean the related settings
func (ctl *Controller) RemovePolicy(policyID int) error {
return nil
func (ctl *Controller) RemovePolicy(policyID int64) error {
// TODO check pre-conditions
return ctl.policyManager.RemovePolicy(policyID)
}
//GetPolicy is delegation of GetPolicy of Policy.Manager
func (ctl *Controller) GetPolicy(policyID int) models.ReplicationPolicy {
return models.ReplicationPolicy{}
func (ctl *Controller) GetPolicy(policyID int64) (models.ReplicationPolicy, error) {
return ctl.policyManager.GetPolicy(policyID)
}
//GetPolicies is delegation of GetPolicies of Policy.Manager
func (ctl *Controller) GetPolicies(query models.QueryParameter) []models.ReplicationPolicy {
return nil
//GetPolicies is delegation of GetPoliciemodels.ReplicationPolicy{}s of Policy.Manager
func (ctl *Controller) GetPolicies(query models.QueryParameter) ([]models.ReplicationPolicy, error) {
return ctl.policyManager.GetPolicies(query)
}
//Replicate starts one replication defined in the specified policy;
//Can be launched by the API layer and related triggers.
func (ctl *Controller) Replicate(policyID int) error {
func (ctl *Controller) Replicate(policyID int64) error {
return nil
}

5
src/replication/event/on_deletion_handler.go
View File

@ -36,7 +36,10 @@ func (oph *OnDeletionHandler) Handle(value interface{}) error {
ProjectID: 0,
}
policies := core.DefaultController.GetPolicies(query)
policies, err := core.DefaultController.GetPolicies(query)
if err != nil {
return err
}
if policies != nil && len(policies) > 0 {
for _, p := range policies {
//Error accumulated and then return?

5
src/replication/event/on_push_handler.go
View File

@ -36,7 +36,10 @@ func (oph *OnPushHandler) Handle(value interface{}) error {
ProjectID: 0,
}
policies := core.DefaultController.GetPolicies(query)
policies, err := core.DefaultController.GetPolicies(query)
if err != nil {
return err
}
if policies != nil && len(policies) > 0 {
for _, p := range policies {
if err := core.DefaultController.Replicate(p.ID); err != nil {

2
src/replication/event/start_replication_handler.go
View File

@ -14,7 +14,7 @@ type StartReplicationHandler struct{}
//StartReplicationNotification contains data required by this handler
type StartReplicationNotification struct {
//ID of the policy
PolicyID int
PolicyID int64
}
//Handle implements the same method of notification handler interface

26
src/replication/models/filter_item.go
View File

@ -1,19 +1,39 @@
package models
import (
"fmt"
"github.com/astaxie/beego/validation"
"github.com/vmware/harbor/src/replication"
)
//FilterItem is the general data model represents the filtering resources which are used as input and output for the filters.
type FilterItem struct {
//The kind of the filtering resources. Support 'project','repository' and 'tag' etc.
Kind string
Kind string `json:"kind"`
//The key value of resource which can be used to filter out the resource matched with specified pattern.
//E.g:
//kind == 'project', value will be project name;
//kind == 'repository', value will be repository name
//kind == 'tag', value will be tag name.
Value string
Value string `json:"value"`
//Extension placeholder.
//To append more additional information if required by the filter.
Metadata map[string]interface{}
Metadata map[string]interface{} `json:"metadata"`
}
// Valid ...
func (f *FilterItem) Valid(v *validation.Validation) {
if !(f.Kind == replication.FilterItemKindProject ||
f.Kind == replication.FilterItemKindRepository ||
f.Kind == replication.FilterItemKindTag) {
v.SetError("kind", fmt.Sprintf("invalid filter kind: %s", f.Kind))
}
if len(f.Value) == 0 {
v.SetError("value", "filter value can not be empty")
}
}

31
src/replication/models/policy.go
View File

@ -1,28 +1,37 @@
package models
import (
"time"
)
//ReplicationPolicy defines the structure of a replication policy.
type ReplicationPolicy struct {
//UUID of the policy
ID int
//Projects attached to this policy
RelevantProjects []int
//The trigger of the replication
Trigger Trigger
ID int64 //UUID of the policy
Name string
Description string
Filters []FilterItem
ReplicateDeletion bool
Trigger *Trigger //The trigger of the replication
ProjectIDs []int64 //Projects attached to this policy
TargetIDs []int64
CreationTime time.Time
UpdateTime time.Time
}
//QueryParameter defines the parameters used to do query selection.
type QueryParameter struct {
//Query by page, couple with pageSize
Page int
Page int64
//Size of each page, couple with page
PageSize int
PageSize int64
//Query by the name of trigger
TriggerName string
//Query by project ID
ProjectID int
ProjectID int64
//Query by name
Name string
}

20
src/replication/models/trigger.go
View File

@ -1,10 +1,26 @@
package models
import (
"fmt"
"github.com/astaxie/beego/validation"
"github.com/vmware/harbor/src/replication"
)
//Trigger is replication launching approach definition
type Trigger struct {
//The name of the trigger
Name string
Kind string `json:"kind"`
//The parameters with json text format required by the trigger
Param string
Param string `json:"param"`
}
// Valid ...
func (t *Trigger) Valid(v *validation.Validation) {
if !(t.Kind == replication.TriggerKindImmediately ||
t.Kind == replication.TriggerKindManually ||
t.Kind == replication.TriggerKindSchedule) {
v.SetError("kind", fmt.Sprintf("invalid trigger kind: %s", t.Kind))
}
}

129
src/replication/policy/manager.go
View File

@ -1,6 +1,11 @@
package policy
import (
"encoding/json"
"time"
"github.com/vmware/harbor/src/common/dao"
persist_models "github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/replication/models"
)
@ -13,30 +18,136 @@ func NewManager() *Manager {
}
//GetPolicies returns all the policies
func (m *Manager) GetPolicies(query models.QueryParameter) []models.ReplicationPolicy {
return []models.ReplicationPolicy{}
func (m *Manager) GetPolicies(query models.QueryParameter) ([]models.ReplicationPolicy, error) {
result := []models.ReplicationPolicy{}
//TODO support more query conditions other than name and project ID
policies, err := dao.FilterRepPolicies(query.Name, query.ProjectID)
if err != nil {
return result, err
}
for _, policy := range policies {
ply, err := convertFromPersistModel(policy)
if err != nil {
return []models.ReplicationPolicy{}, err
}
result = append(result, ply)
}
return result, nil
}
//GetPolicy returns the policy with the specified ID
func (m *Manager) GetPolicy(policyID int) models.ReplicationPolicy {
return models.ReplicationPolicy{}
func (m *Manager) GetPolicy(policyID int64) (models.ReplicationPolicy, error) {
policy, err := dao.GetRepPolicy(policyID)
if err != nil {
return models.ReplicationPolicy{}, err
}
return convertFromPersistModel(policy)
}
// TODO add UT
func convertFromPersistModel(policy *persist_models.RepPolicy) (models.ReplicationPolicy, error) {
if policy == nil {
return models.ReplicationPolicy{}, nil
}
ply := models.ReplicationPolicy{
ID: policy.ID,
Name: policy.Name,
Description: policy.Description,
ReplicateDeletion: policy.ReplicateDeletion,
ProjectIDs: []int64{policy.ProjectID},
TargetIDs: []int64{policy.TargetID},
CreationTime: policy.CreationTime,
UpdateTime: policy.UpdateTime,
}
if len(policy.Filters) > 0 {
filters := []models.FilterItem{}
if err := json.Unmarshal([]byte(policy.Filters), &filters); err != nil {
return models.ReplicationPolicy{}, err
}
ply.Filters = filters
}
if len(policy.Trigger) > 0 {
trigger := &models.Trigger{}
if err := json.Unmarshal([]byte(policy.Trigger), trigger); err != nil {
return models.ReplicationPolicy{}, err
}
ply.Trigger = trigger
}
return ply, nil
}
// TODO add ut
func convertToPersistModel(policy models.ReplicationPolicy) (*persist_models.RepPolicy, error) {
ply := &persist_models.RepPolicy{
ID: policy.ID,
Name: policy.Name,
Description: policy.Description,
ReplicateDeletion: policy.ReplicateDeletion,
CreationTime: policy.CreationTime,
UpdateTime: policy.UpdateTime,
}
if len(policy.ProjectIDs) > 0 {
ply.ProjectID = policy.ProjectIDs[0]
}
if len(policy.TargetIDs) > 0 {
ply.TargetID = policy.TargetIDs[0]
}
if policy.Trigger != nil {
trigger, err := json.Marshal(policy.Trigger)
if err != nil {
return nil, err
}
ply.Trigger = string(trigger)
}
if len(policy.Filters) > 0 {
filters, err := json.Marshal(policy.Filters)
if err != nil {
return nil, err
}
ply.Filters = string(filters)
}
return ply, nil
}
//CreatePolicy creates a new policy with the provided data;
//If creating failed, error will be returned;
//If creating succeed, ID of the new created policy will be returned.
func (m *Manager) CreatePolicy(policy models.ReplicationPolicy) (int, error) {
return 0, nil
func (m *Manager) CreatePolicy(policy models.ReplicationPolicy) (int64, error) {
now := time.Now()
policy.CreationTime = now
policy.UpdateTime = now
ply, err := convertToPersistModel(policy)
if err != nil {
return 0, err
}
return dao.AddRepPolicy(*ply)
}
//UpdatePolicy updates the policy;
//If updating failed, error will be returned.
func (m *Manager) UpdatePolicy(policy models.ReplicationPolicy) error {
return nil
policy.UpdateTime = time.Now()
ply, err := convertToPersistModel(policy)
if err != nil {
return err
}
return dao.UpdateRepPolicy(ply)
}
//RemovePolicy removes the specified policy;
//If removing failed, error will be returned.
func (m *Manager) RemovePolicy(policyID int) error {
return nil
func (m *Manager) RemovePolicy(policyID int64) error {
return dao.DeleteRepPolicy(policyID)
}

10
src/replication/trigger/cache.go
View File

@ -15,7 +15,7 @@ const (
//Item keeps more metadata of the triggers which are stored in the heap.
type Item struct {
//Which policy the trigger belong to
policyID int
policyID int64
//Frequency of cache querying
//First compration factor
@ -124,7 +124,7 @@ func NewCache(capacity int) *Cache {
}
//Get the trigger interface with the specified policy ID
func (c *Cache) Get(policyID int) Interface {
func (c *Cache) Get(policyID int64) Interface {
if policyID <= 0 {
return nil
}
@ -144,7 +144,7 @@ func (c *Cache) Get(policyID int) Interface {
}
//Put the item into cache with ID of ploicy as key
func (c *Cache) Put(policyID int, trigger Interface) {
func (c *Cache) Put(policyID int64, trigger Interface) {
if policyID <= 0 || trigger == nil {
return
}
@ -179,7 +179,7 @@ func (c *Cache) Put(policyID int, trigger Interface) {
}
//Remove the trigger attached to the specified policy
func (c *Cache) Remove(policyID int) Interface {
func (c *Cache) Remove(policyID int64) Interface {
if policyID > 0 {
c.lock.Lock()
defer c.lock.Unlock()
@ -207,6 +207,6 @@ func (c *Cache) Size() int {
}
//Generate a hash key with the policy ID
func (c *Cache) key(policyID int) string {
func (c *Cache) key(policyID int64) string {
return fmt.Sprintf("trigger-%d", policyID)
}

16
src/replication/trigger/manager.go
View File

@ -24,12 +24,12 @@ func NewManager(capacity int) *Manager {
}
//GetTrigger returns the enabled trigger reference if existing in the cache.
func (m *Manager) GetTrigger(policyID int) Interface {
func (m *Manager) GetTrigger(policyID int64) Interface {
return m.cache.Get(policyID)
}
//RemoveTrigger will disable the trigger and remove it from the cache if existing.
func (m *Manager) RemoveTrigger(policyID int) error {
func (m *Manager) RemoveTrigger(policyID int64) error {
trigger := m.cache.Get(policyID)
if trigger == nil {
return errors.New("Trigger is not cached, please use UnsetTrigger to disable the trigger")
@ -50,16 +50,16 @@ func (m *Manager) RemoveTrigger(policyID int) error {
//SetupTrigger will create the new trigger based on the provided json parameters.
//If failed, an error will be returned.
func (m *Manager) SetupTrigger(policyID int, trigger models.Trigger) error {
func (m *Manager) SetupTrigger(policyID int64, trigger models.Trigger) error {
if policyID <= 0 {
return errors.New("Invalid policy ID")
}
if len(trigger.Name) == 0 {
if len(trigger.Kind) == 0 {
return errors.New("Invalid replication trigger definition")
}
switch trigger.Name {
switch trigger.Kind {
case replication.TriggerKindSchedule:
param := ScheduleParam{}
if err := param.Parse(trigger.Param); err != nil {
@ -93,16 +93,16 @@ func (m *Manager) SetupTrigger(policyID int, trigger models.Trigger) error {
}
//UnsetTrigger will disable the trigger which is not cached in the trigger cache.
func (m *Manager) UnsetTrigger(policyID int, trigger models.Trigger) error {
func (m *Manager) UnsetTrigger(policyID int64, trigger models.Trigger) error {
if policyID <= 0 {
return errors.New("Invalid policy ID")
}
if len(trigger.Name) == 0 {
if len(trigger.Kind) == 0 {
return errors.New("Invalid replication trigger definition")
}
switch trigger.Name {
switch trigger.Kind {
case replication.TriggerKindSchedule:
param := ScheduleParam{}
if err := param.Parse(trigger.Param); err != nil {

2
src/replication/trigger/trigger_param.go
View File

@ -3,7 +3,7 @@ package trigger
//BasicParam contains the general parameters for all triggers
type BasicParam struct {
//ID of the related policy
PolicyID int
PolicyID int64
//Whether delete remote replicated images if local ones are deleted
OnDeletion bool

2
src/replication/trigger/watch_list.go
View File

@ -10,7 +10,7 @@ type WatchList struct{}
//WatchItem keeps the related data for evaluation in WatchList.
type WatchItem struct {
//ID of policy
PolicyID int
PolicyID int64
//Corresponding namespace
Namespace string

195
src/ui/api/api_test.go Normal file
View File

@ -0,0 +1,195 @@
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"strconv"
"strings"
"testing"
"github.com/astaxie/beego"
"github.com/dghubble/sling"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
)
var (
nonSysAdminID int64
sysAdmin = &usrInfo{
Name: "admin",
Passwd: "Harbor12345",
}
nonSysAdmin = &usrInfo{
Name: "non_admin",
Passwd: "Harbor12345",
}
)
type testingRequest struct {
method string
url string
header http.Header
queryStruct interface{}
bodyJSON interface{}
credential *usrInfo
}
type codeCheckingCase struct {
request *testingRequest
code int
postFunc func(*httptest.ResponseRecorder) error
}
func newRequest(r *testingRequest) (*http.Request, error) {
if r == nil {
return nil, nil
}
reqBuilder := sling.New()
switch strings.ToUpper(r.method) {
case "", http.MethodGet:
reqBuilder = reqBuilder.Get(r.url)
case http.MethodPost:
reqBuilder = reqBuilder.Post(r.url)
case http.MethodPut:
reqBuilder = reqBuilder.Put(r.url)
case http.MethodDelete:
reqBuilder = reqBuilder.Delete(r.url)
case http.MethodHead:
reqBuilder = reqBuilder.Head(r.url)
case http.MethodPatch:
reqBuilder = reqBuilder.Patch(r.url)
default:
return nil, fmt.Errorf("unsupported method %s", r.method)
}
for key, values := range r.header {
for _, value := range values {
reqBuilder = reqBuilder.Add(key, value)
}
}
if r.queryStruct != nil {
reqBuilder = reqBuilder.QueryStruct(r.queryStruct)
}
if r.bodyJSON != nil {
reqBuilder = reqBuilder.BodyJSON(r.bodyJSON)
}
if r.credential != nil {
reqBuilder = reqBuilder.SetBasicAuth(r.credential.Name, r.credential.Passwd)
}
return reqBuilder.Request()
}
func handle(r *testingRequest) (*httptest.ResponseRecorder, error) {
req, err := newRequest(r)
if err != nil {
return nil, err
}
resp := httptest.NewRecorder()
beego.BeeApp.Handlers.ServeHTTP(resp, req)
return resp, nil
}
func handleAndParse(r *testingRequest, v interface{}) (*httptest.ResponseRecorder, error) {
req, err := newRequest(r)
if err != nil {
return nil, err
}
resp := httptest.NewRecorder()
beego.BeeApp.Handlers.ServeHTTP(resp, req)
if resp.Code >= 200 && resp.Code <= 299 {
if err := json.NewDecoder(resp.Body).Decode(v); err != nil {
return nil, err
}
}
return resp, nil
}
func runCodeCheckingCases(t *testing.T, cases ...*codeCheckingCase) {
for _, c := range cases {
resp, err := handle(c.request)
require.Nil(t, err)
equal := assert.Equal(t, c.code, resp.Code)
if !equal {
if resp.Body.Len() > 0 {
t.Log(resp.Body.String())
}
continue
}
if c.postFunc != nil {
if err := c.postFunc(resp); err != nil {
t.Logf("error in running post function: %v", err)
}
}
}
}
func parseResourceID(resp *httptest.ResponseRecorder) (int64, error) {
location := resp.Header().Get(http.CanonicalHeaderKey("location"))
if len(location) == 0 {
return 0, fmt.Errorf("empty location header")
}
index := strings.LastIndex(location, "/")
if index == -1 {
return 0, fmt.Errorf("location header %s contains no /", location)
}
id := strings.TrimPrefix(location, location[:index+1])
if len(id) == 0 {
return 0, fmt.Errorf("location header %s contains no resource ID", location)
}
return strconv.ParseInt(id, 10, 64)
}
func TestMain(m *testing.M) {
if err := prepare(); err != nil {
panic(err)
}
defer clean()
os.Exit(m.Run())
}
func prepare() error {
id, err := dao.Register(models.User{
Username: nonSysAdmin.Name,
Password: nonSysAdmin.Passwd,
})
if err != nil {
return err
}
nonSysAdminID = id
return nil
}
func clean() error {
return dao.DeleteUser(int(nonSysAdminID))
}

1
src/ui/api/harborapi_test.go
View File

@ -122,7 +122,6 @@ func init() {
beego.Router("/api/policies/replication/:id([0-9]+)", &RepPolicyAPI{})
beego.Router("/api/policies/replication", &RepPolicyAPI{}, "get:List")
beego.Router("/api/policies/replication", &RepPolicyAPI{}, "post:Post;delete:Delete")
beego.Router("/api/policies/replication/:id([0-9]+)/enablement", &RepPolicyAPI{}, "put:UpdateEnablement")
beego.Router("/api/systeminfo", &SystemInfoAPI{}, "get:GetGeneralInfo")
beego.Router("/api/systeminfo/volumes", &SystemInfoAPI{}, "get:GetVolumeInfo")
beego.Router("/api/systeminfo/getcert", &SystemInfoAPI{}, "get:GetCert")

66
src/ui/api/models/replication_policy.go Normal file
View File

@ -0,0 +1,66 @@
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"time"
"github.com/astaxie/beego/validation"
common_models "github.com/vmware/harbor/src/common/models"
rep_models "github.com/vmware/harbor/src/replication/models"
)
// ReplicationPolicy defines the data model used in API level
type ReplicationPolicy struct {
ID int64 `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Filters []rep_models.FilterItem `json:"filters"`
ReplicateDeletion bool `json:"replicate_deletion"`
Trigger *rep_models.Trigger `json:"trigger"`
Projects []*common_models.Project `json:"projects"`
Targets []*common_models.RepTarget `json:"targets"`
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
ReplicateExistingImageNow bool `json:"replicate_existing_image_now"`
ErrorJobCount int64 `json:"error_job_count"`
}
// Valid ...
func (r *ReplicationPolicy) Valid(v *validation.Validation) {
if len(r.Name) == 0 {
v.SetError("name", "can not be empty")
}
if len(r.Name) > 256 {
v.SetError("name", "max length is 256")
}
if len(r.Projects) == 0 {
v.SetError("projects", "can not be empty")
}
if len(r.Targets) == 0 {
v.SetError("targets", "can not be empty")
}
for _, filter := range r.Filters {
filter.Valid(v)
}
if r.Trigger != nil {
r.Trigger.Valid(v)
}
}

414
src/ui/api/replication_policy.go
View File

@ -23,6 +23,10 @@ import (
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/replication/core"
rep_models "github.com/vmware/harbor/src/replication/models"
api_models "github.com/vmware/harbor/src/ui/api/models"
"github.com/vmware/harbor/src/ui/promgr"
)
// RepPolicyAPI handles /api/replicationPolicies /api/replicationPolicies/:id/enablement
@ -47,344 +51,144 @@ func (pa *RepPolicyAPI) Prepare() {
// Get ...
func (pa *RepPolicyAPI) Get() {
id := pa.GetIDFromURL()
policy, err := dao.GetRepPolicy(id)
policy, err := core.DefaultController.GetPolicy(id)
if err != nil {
log.Errorf("failed to get policy %d: %v", id, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if policy == nil {
if policy.ID == 0 {
pa.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound))
}
pa.Data["json"] = policy
ply, err := convertFromRepPolicy(pa.ProjectMgr, policy)
if err != nil {
pa.ParseAndHandleError(fmt.Sprintf("failed to convert from replication policy"), err)
return
}
pa.Data["json"] = ply
pa.ServeJSON()
}
// List filters policies by name and project_id, if name and project_id
// are nil, List returns all policies
// List ...
func (pa *RepPolicyAPI) List() {
name := pa.GetString("name")
queryParam := rep_models.QueryParameter{
Name: pa.GetString("name"),
}
projectIDStr := pa.GetString("project_id")
var projectID int64
var err error
if len(projectIDStr) != 0 {
projectID, err = strconv.ParseInt(projectIDStr, 10, 64)
if len(projectIDStr) > 0 {
projectID, err := strconv.ParseInt(projectIDStr, 10, 64)
if err != nil || projectID <= 0 {
pa.CustomAbort(http.StatusBadRequest, "invalid project ID")
}
queryParam.ProjectID = projectID
}
policies, err := dao.FilterRepPolicies(name, projectID)
result := []*api_models.ReplicationPolicy{}
policies, err := core.DefaultController.GetPolicies(queryParam)
if err != nil {
log.Errorf("failed to filter policies %s project ID %d: %v", name, projectID, err)
log.Errorf("failed to get policies: %v, query parameters: %v", err, queryParam)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
for _, policy := range policies {
project, err := pa.ProjectMgr.Get(policy.ProjectID)
ply, err := convertFromRepPolicy(pa.ProjectMgr, policy)
if err != nil {
pa.ParseAndHandleError(fmt.Sprintf(
"failed to get project %d", policy.ProjectID), err)
pa.ParseAndHandleError(fmt.Sprintf("failed to convert from replication policy"), err)
return
}
if project != nil {
policy.ProjectName = project.Name
}
result = append(result, ply)
}
pa.Data["json"] = policies
pa.Data["json"] = result
pa.ServeJSON()
}
// Post creates a policy, and if it is enbled, the replication will be triggered right now.
// Post creates a replicartion policy
func (pa *RepPolicyAPI) Post() {
policy := &models.RepPolicy{}
policy := &api_models.ReplicationPolicy{}
pa.DecodeJSONReqAndValidate(policy)
/*
po, err := dao.GetRepPolicyByName(policy.Name)
// check the existence of projects
for _, project := range policy.Projects {
exist, err := pa.ProjectMgr.Exists(project.ProjectID)
if err != nil {
log.Errorf("failed to get policy %s: %v", policy.Name, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
pa.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d", project.ProjectID), err)
return
}
if !exist {
pa.HandleNotFound(fmt.Sprintf("project %d not found", project.ProjectID))
return
}
}
// check the existence of targets
for _, target := range policy.Targets {
t, err := dao.GetRepTarget(target.ID)
if err != nil {
pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", target.ID, err))
return
}
if po != nil {
pa.CustomAbort(http.StatusConflict, "name is already used")
if t == nil {
pa.HandleNotFound(fmt.Sprintf("target %d not found", target.ID))
return
}
*/
}
project, err := pa.ProjectMgr.Get(policy.ProjectID)
id, err := core.DefaultController.CreatePolicy(convertToRepPolicy(policy))
if err != nil {
pa.ParseAndHandleError(fmt.Sprintf("failed to get project %d", policy.ProjectID), err)
pa.HandleInternalServerError(fmt.Sprintf("failed to create policy: %v", err))
return
}
if project == nil {
pa.CustomAbort(http.StatusBadRequest, fmt.Sprintf("project %d does not exist", policy.ProjectID))
}
// TODO trigger a replication if ReplicateExistingImageNow is true
target, err := dao.GetRepTarget(policy.TargetID)
if err != nil {
log.Errorf("failed to get target %d: %v", policy.TargetID, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if target == nil {
pa.CustomAbort(http.StatusBadRequest, fmt.Sprintf("target %d does not exist", policy.TargetID))
}
policies, err := dao.GetRepPolicyByProjectAndTarget(policy.ProjectID, policy.TargetID)
if err != nil {
log.Errorf("failed to get policy [project ID: %d,targetID: %d]: %v", policy.ProjectID, policy.TargetID, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if len(policies) > 0 {
pa.CustomAbort(http.StatusConflict, "policy already exists with the same project and target")
}
pid, err := dao.AddRepPolicy(*policy)
if err != nil {
log.Errorf("Failed to add policy to DB, error: %v", err)
pa.RenderError(http.StatusInternalServerError, "Internal Error")
return
}
if policy.Enabled == 1 {
go func() {
if err := TriggerReplication(pid, "", nil, models.RepOpTransfer); err != nil {
log.Errorf("failed to trigger replication of %d: %v", pid, err)
} else {
log.Infof("replication of %d triggered", pid)
}
}()
}
pa.Redirect(http.StatusCreated, strconv.FormatInt(pid, 10))
pa.Redirect(http.StatusCreated, strconv.FormatInt(id, 10))
}
// Put modifies name, description, target and enablement of policy
// Put updates the replication policy
func (pa *RepPolicyAPI) Put() {
id := pa.GetIDFromURL()
originalPolicy, err := dao.GetRepPolicy(id)
originalPolicy, err := core.DefaultController.GetPolicy(id)
if err != nil {
log.Errorf("failed to get policy %d: %v", id, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if originalPolicy == nil {
if originalPolicy.ID == 0 {
pa.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound))
}
policy := &models.RepPolicy{}
pa.DecodeJSONReq(policy)
policy.ProjectID = originalPolicy.ProjectID
pa.Validate(policy)
/*
// check duplicate name
if policy.Name != originalPolicy.Name {
po, err := dao.GetRepPolicyByName(policy.Name)
if err != nil {
log.Errorf("failed to get policy %s: %v", policy.Name, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if po != nil {
pa.CustomAbort(http.StatusConflict, "name is already used")
}
}
*/
if policy.TargetID != originalPolicy.TargetID {
//target of policy can not be modified when the policy is enabled
if originalPolicy.Enabled == 1 {
pa.CustomAbort(http.StatusBadRequest, "target of policy can not be modified when the policy is enabled")
}
// check the existance of target
target, err := dao.GetRepTarget(policy.TargetID)
if err != nil {
log.Errorf("failed to get target %d: %v", policy.TargetID, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if target == nil {
pa.CustomAbort(http.StatusBadRequest, fmt.Sprintf("target %d does not exist", policy.TargetID))
}
// check duplicate policy with the same project and target
policies, err := dao.GetRepPolicyByProjectAndTarget(policy.ProjectID, policy.TargetID)
if err != nil {
log.Errorf("failed to get policy [project ID: %d,targetID: %d]: %v", policy.ProjectID, policy.TargetID, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if len(policies) > 0 {
pa.CustomAbort(http.StatusConflict, "policy already exists with the same project and target")
}
}
policy := &api_models.ReplicationPolicy{}
pa.DecodeJSONReqAndValidate(policy)
policy.ID = id
/*
isTargetChanged := !(policy.TargetID == originalPolicy.TargetID)
isEnablementChanged := !(policy.Enabled == policy.Enabled)
var shouldStop, shouldTrigger bool
// if target and enablement are not changed, do nothing
if !isTargetChanged && !isEnablementChanged {
shouldStop = false
shouldTrigger = false
} else if !isTargetChanged && isEnablementChanged {
// target is not changed, but enablement is changed
if policy.Enabled == 0 {
shouldStop = true
shouldTrigger = false
} else {
shouldStop = false
shouldTrigger = true
}
} else if isTargetChanged && !isEnablementChanged {
// target is changed, but enablement is not changed
if policy.Enabled == 0 {
// enablement is 0, do nothing
shouldStop = false
shouldTrigger = false
} else {
// enablement is 1, so stop original target's jobs
// and trigger new target's jobs
shouldStop = true
shouldTrigger = true
}
} else {
// both target and enablement are changed
// enablement: 1 -> 0
if policy.Enabled == 0 {
shouldStop = true
shouldTrigger = false
} else {
shouldStop = false
shouldTrigger = true
}
}
if shouldStop {
if err := postReplicationAction(id, "stop"); err != nil {
log.Errorf("failed to stop replication of %d: %v", id, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
log.Infof("replication of %d has been stopped", id)
}
if err = dao.UpdateRepPolicy(policy); err != nil {
log.Errorf("failed to update policy %d: %v", id, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if shouldTrigger {
go func() {
if err := TriggerReplication(id, "", nil, models.RepOpTransfer); err != nil {
log.Errorf("failed to trigger replication of %d: %v", id, err)
} else {
log.Infof("replication of %d triggered", id)
}
}()
}
*/
if err = dao.UpdateRepPolicy(policy); err != nil {
log.Errorf("failed to update policy %d: %v", id, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if policy.Enabled != originalPolicy.Enabled && policy.Enabled == 1 {
go func() {
if err := TriggerReplication(id, "", nil, models.RepOpTransfer); err != nil {
log.Errorf("failed to trigger replication of %d: %v", id, err)
} else {
log.Infof("replication of %d triggered", id)
}
}()
if err = core.DefaultController.UpdatePolicy(convertToRepPolicy(policy)); err != nil {
pa.HandleInternalServerError(fmt.Sprintf("failed to update policy %d: %v", id, err))
return
}
}
type enablementReq struct {
Enabled int `json:"enabled"`
}
// UpdateEnablement changes the enablement of the policy
func (pa *RepPolicyAPI) UpdateEnablement() {
// Delete the replication policy
func (pa *RepPolicyAPI) Delete() {
id := pa.GetIDFromURL()
policy, err := dao.GetRepPolicy(id)
policy, err := core.DefaultController.GetPolicy(id)
if err != nil {
log.Errorf("failed to get policy %d: %v", id, err)
pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if policy == nil {
if policy.ID == 0 {
pa.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound))
}
e := enablementReq{}
pa.DecodeJSONReq(&e)
if e.Enabled != 0 && e.Enabled != 1 {
pa.RenderError(http.StatusBadRequest, "invalid enabled value")
return
}
if policy.Enabled == e.Enabled {
return
}
if err := dao.UpdateRepPolicyEnablement(id, e.Enabled); err != nil {
log.Errorf("Failed to update policy enablement in DB, error: %v", err)
pa.RenderError(http.StatusInternalServerError, "Internal Error")
return
}
if e.Enabled == 1 {
go func() {
if err := TriggerReplication(id, "", nil, models.RepOpTransfer); err != nil {
log.Errorf("failed to trigger replication of %d: %v", id, err)
} else {
log.Infof("replication of %d triggered", id)
}
}()
} else {
go func() {
if err := postReplicationAction(id, "stop"); err != nil {
log.Errorf("failed to stop replication of %d: %v", id, err)
} else {
log.Infof("try to stop replication of %d", id)
}
}()
}
}
// Delete : policies which are disabled and have no running jobs
// can be deleted
func (pa *RepPolicyAPI) Delete() {
id := pa.GetIDFromURL()
policy, err := dao.GetRepPolicy(id)
if err != nil {
log.Errorf("failed to get policy %d: %v", id, err)
pa.CustomAbort(http.StatusInternalServerError, "")
}
if policy == nil || policy.Deleted == 1 {
pa.CustomAbort(http.StatusNotFound, "")
}
if policy.Enabled == 1 {
pa.CustomAbort(http.StatusPreconditionFailed, "plicy is enabled, can not be deleted")
}
// TODO
jobs, err := dao.GetRepJobByPolicy(id)
if err != nil {
log.Errorf("failed to get jobs of policy %d: %v", id, err)
@ -399,8 +203,82 @@ func (pa *RepPolicyAPI) Delete() {
}
}
if err = dao.DeleteRepPolicy(id); err != nil {
if err = core.DefaultController.RemovePolicy(id); err != nil {
log.Errorf("failed to delete policy %d: %v", id, err)
pa.CustomAbort(http.StatusInternalServerError, "")
}
}
func convertFromRepPolicy(projectMgr promgr.ProjectManager, policy rep_models.ReplicationPolicy) (*api_models.ReplicationPolicy, error) {
if policy.ID == 0 {
return nil, nil
}
// populate simple properties
ply := &api_models.ReplicationPolicy{
ID: policy.ID,
Name: policy.Name,
Description: policy.Description,
Filters: policy.Filters,
ReplicateDeletion: policy.ReplicateDeletion,
Trigger: policy.Trigger,
CreationTime: policy.CreationTime,
UpdateTime: policy.UpdateTime,
}
// populate projects
for _, projectID := range policy.ProjectIDs {
project, err := projectMgr.Get(projectID)
if err != nil {
return nil, err
}
ply.Projects = append(ply.Projects, project)
}
// populate targets
for _, targetID := range policy.TargetIDs {
target, err := dao.GetRepTarget(targetID)
if err != nil {
return nil, err
}
target.Password = ""
ply.Targets = append(ply.Targets, target)
}
// TODO call the method from replication controller
_, errJobCount, err := dao.FilterRepJobs(policy.ID, "", "error", nil, nil, 0, 0)
if err != nil {
return nil, err
}
ply.ErrorJobCount = errJobCount
return ply, nil
}
func convertToRepPolicy(policy *api_models.ReplicationPolicy) rep_models.ReplicationPolicy {
if policy == nil {
return rep_models.ReplicationPolicy{}
}
ply := rep_models.ReplicationPolicy{
ID: policy.ID,
Name: policy.Name,
Description: policy.Description,
Filters: policy.Filters,
ReplicateDeletion: policy.ReplicateDeletion,
Trigger: policy.Trigger,
CreationTime: policy.CreationTime,
UpdateTime: policy.UpdateTime,
}
for _, project := range policy.Projects {
ply.ProjectIDs = append(ply.ProjectIDs, project.ProjectID)
}
for _, target := range policy.Targets {
ply.TargetIDs = append(ply.TargetIDs, target.ID)
}
return ply
}

722
src/ui/api/replication_policy_test.go
View File

@ -15,296 +15,498 @@ package api
import (
"fmt"
"strconv"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/replication"
"github.com/vmware/harbor/tests/apitests/apilib"
rep_models "github.com/vmware/harbor/src/replication/models"
api_models "github.com/vmware/harbor/src/ui/api/models"
)
const (
addPolicyName = "testPolicy"
var (
repPolicyAPIBasePath = "/api/policies/replication"
policyName = "testPolicy"
projectID int64 = 1
targetID int64
policyID int64
)
var addPolicyID int
func TestRepPolicyAPIPost(t *testing.T) {
postFunc := func(resp *httptest.ResponseRecorder) error {
id, err := parseResourceID(resp)
if err != nil {
return err
}
policyID = id
return nil
}
func TestPoliciesPost(t *testing.T) {
var httpStatusCode int
var err error
assert := assert.New(t)
apiTest := newHarborAPI()
//add target
CommonAddTarget()
targetID := int64(CommonGetTarget())
repPolicy := &apilib.RepPolicyPost{int64(1), targetID, addPolicyName,
&models.RepTrigger{
Type: replication.TriggerKindSchedule,
Params: map[string]interface{}{
"date": "2:00",
targetID = int64(CommonGetTarget())
cases := []*codeCheckingCase{
// 401
&codeCheckingCase{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
},
code: http.StatusUnauthorized,
},
[]*models.RepFilter{
&models.RepFilter{
Type: replication.FilterItemKindRepository,
Value: "library/ubuntu*",
// 403
&codeCheckingCase{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
credential: nonSysAdmin,
},
}}
fmt.Println("Testing Policies Post API")
//-------------------case 1 : response code = 201------------------------//
fmt.Println("case 1 : response code = 201")
httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy)
if err != nil {
t.Error("Error while add policy", err.Error())
t.Log(err)
} else {
assert.Equal(int(201), httpStatusCode, "httpStatusCode should be 201")
}
//-------------------case 2 : response code = 409------------------------//
fmt.Println("case 2 : response code = 409:policy already exists")
httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy)
if err != nil {
t.Error("Error while add policy", err.Error())
t.Log(err)
} else {
assert.Equal(int(409), httpStatusCode, "httpStatusCode should be 409")
}
//-------------------case 3 : response code = 401------------------------//
fmt.Println("case 3 : response code = 401: User need to log in first.")
httpStatusCode, err = apiTest.AddPolicy(*unknownUsr, *repPolicy)
if err != nil {
t.Error("Error while add policy", err.Error())
t.Log(err)
} else {
assert.Equal(int(401), httpStatusCode, "httpStatusCode should be 401")
}
//-------------------case 4 : response code = 400------------------------//
fmt.Println("case 4 : response code = 400:project_id invalid.")
repPolicy = &apilib.RepPolicyPost{TargetId: targetID, Name: addPolicyName}
httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy)
if err != nil {
t.Error("Error while add policy", err.Error())
t.Log(err)
} else {
assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400")
}
//-------------------case 5 : response code = 400------------------------//
fmt.Println("case 5 : response code = 400:project_id does not exist.")
repPolicy.ProjectId = int64(1111)
httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy)
if err != nil {
t.Error("Error while add policy", err.Error())
t.Log(err)
} else {
assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400")
}
//-------------------case 6 : response code = 400------------------------//
fmt.Println("case 6 : response code = 400:target_id invalid.")
repPolicy = &apilib.RepPolicyPost{ProjectId: int64(1), Name: addPolicyName}
httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy)
if err != nil {
t.Error("Error while add policy", err.Error())
t.Log(err)
} else {
assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400")
}
//-------------------case 7 : response code = 400------------------------//
fmt.Println("case 7 : response code = 400:target_id does not exist.")
repPolicy.TargetId = int64(1111)
httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy)
if err != nil {
t.Error("Error while add policy", err.Error())
t.Log(err)
} else {
assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400")
}
fmt.Println("case 8 : response code = 400: invalid filter")
repPolicy = &apilib.RepPolicyPost{int64(1), targetID, addPolicyName,
&models.RepTrigger{
Type: replication.TriggerKindManually,
code: http.StatusForbidden,
},
[]*models.RepFilter{
&models.RepFilter{
Type: "replication",
Value: "",
// 400, invalid name
&codeCheckingCase{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{},
credential: sysAdmin,
},
}}
httpStatusCode, err = apiTest.AddPolicy(*admin, *repPolicy)
code: http.StatusBadRequest,
},
// 400, invalid projects
&codeCheckingCase{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
},
// 400, invalid targets
&codeCheckingCase{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
&models.Project{
ProjectID: projectID,
},
},
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
},
// 400, invalid filters
&codeCheckingCase{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
&models.Project{
ProjectID: projectID,
},
},
Targets: []*models.RepTarget{
&models.RepTarget{
ID: targetID,
},
},
Filters: []rep_models.FilterItem{
rep_models.FilterItem{
Kind: "invalid_filter_kind",
Value: "",
},
},
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
},
// 400, invalid trigger
&codeCheckingCase{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
&models.Project{
ProjectID: projectID,
},
},
Targets: []*models.RepTarget{
&models.RepTarget{
ID: targetID,
},
},
Filters: []rep_models.FilterItem{
rep_models.FilterItem{
Kind: replication.FilterItemKindRepository,
Value: "*",
},
},
Trigger: &rep_models.Trigger{
Kind: "invalid_trigger_kind",
},
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
},
// 404, project not found
&codeCheckingCase{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
&models.Project{
ProjectID: 10000,
},
},
Targets: []*models.RepTarget{
&models.RepTarget{
ID: targetID,
},
},
Filters: []rep_models.FilterItem{
rep_models.FilterItem{
Kind: replication.FilterItemKindRepository,
Value: "*",
},
},
Trigger: &rep_models.Trigger{
Kind: replication.TriggerKindManually,
},
},
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 404, target not found
&codeCheckingCase{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
&models.Project{
ProjectID: projectID,
},
},
Targets: []*models.RepTarget{
&models.RepTarget{
ID: 10000,
},
},
Filters: []rep_models.FilterItem{
rep_models.FilterItem{
Kind: replication.FilterItemKindRepository,
Value: "*",
},
},
Trigger: &rep_models.Trigger{
Kind: replication.TriggerKindManually,
},
},
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 201
&codeCheckingCase{
request: &testingRequest{
method: http.MethodPost,
url: repPolicyAPIBasePath,
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
&models.Project{
ProjectID: projectID,
},
},
Targets: []*models.RepTarget{
&models.RepTarget{
ID: targetID,
},
},
Filters: []rep_models.FilterItem{
rep_models.FilterItem{
Kind: replication.FilterItemKindRepository,
Value: "*",
},
},
Trigger: &rep_models.Trigger{
Kind: replication.TriggerKindManually,
},
},
credential: sysAdmin,
},
code: http.StatusCreated,
postFunc: postFunc,
},
}
runCodeCheckingCases(t, cases...)
}
func TestRepPolicyAPIGet(t *testing.T) {
// 404
runCodeCheckingCases(t, &codeCheckingCase{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, 10000),
credential: sysAdmin,
},
code: http.StatusNotFound,
})
// 200
policy := &api_models.ReplicationPolicy{}
resp, err := handleAndParse(
&testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID),
credential: sysAdmin,
}, policy)
require.Nil(t, err)
assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400")
assert.Equal(t, http.StatusOK, resp.Code)
assert.Equal(t, policyID, policy.ID)
assert.Equal(t, policyName, policy.Name)
}
func TestPoliciesList(t *testing.T) {
var httpStatusCode int
var err error
var reslut []apilib.RepPolicy
func TestRepPolicyAPIList(t *testing.T) {
// 400: invalid project ID
runCodeCheckingCases(t, &codeCheckingCase{
request: &testingRequest{
method: http.MethodGet,
url: repPolicyAPIBasePath,
queryStruct: struct {
ProjectID int64 `url:"project_id"`
}{
ProjectID: -1,
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
})
assert := assert.New(t)
apiTest := newHarborAPI()
fmt.Println("Testing Policies Get/List API")
//-------------------case 1 : response code = 200------------------------//
fmt.Println("case 1 : response code = 200")
projectID := "1"
httpStatusCode, reslut, err = apiTest.ListPolicies(*admin, addPolicyName, projectID)
if err != nil {
t.Error("Error while get policies", err.Error())
t.Log(err)
} else {
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
addPolicyID = int(reslut[0].Id)
}
//-------------------case 2 : response code = 400------------------------//
fmt.Println("case 2 : response code = 400:invalid projectID")
projectID = "cc"
httpStatusCode, reslut, err = apiTest.ListPolicies(*admin, addPolicyName, projectID)
if err != nil {
t.Error("Error while get policies", err.Error())
t.Log(err)
} else {
assert.Equal(int(400), httpStatusCode, "httpStatusCode should be 400")
}
// 200
policies := []*api_models.ReplicationPolicy{}
resp, err := handleAndParse(
&testingRequest{
method: http.MethodGet,
url: repPolicyAPIBasePath,
queryStruct: struct {
ProjectID int64 `url:"project_id"`
Name string `url:"name"`
}{
ProjectID: projectID,
Name: policyName,
},
credential: sysAdmin,
}, &policies)
require.Nil(t, err)
assert.Equal(t, http.StatusOK, resp.Code)
require.Equal(t, 1, len(policies))
assert.Equal(t, policyID, policies[0].ID)
assert.Equal(t, policyName, policies[0].Name)
// 200
policies = []*api_models.ReplicationPolicy{}
resp, err = handleAndParse(
&testingRequest{
method: http.MethodGet,
url: repPolicyAPIBasePath,
queryStruct: struct {
ProjectID int64 `url:"project_id"`
Name string `url:"name"`
}{
ProjectID: projectID,
Name: "non_exist_policy",
},
credential: sysAdmin,
}, &policies)
require.Nil(t, err)
assert.Equal(t, http.StatusOK, resp.Code)
require.Equal(t, 0, len(policies))
}
func TestPolicyGet(t *testing.T) {
var httpStatusCode int
var err error
assert := assert.New(t)
apiTest := newHarborAPI()
fmt.Println("Testing Policy Get API by PolicyID")
//-------------------case 1 : response code = 200------------------------//
fmt.Println("case 1 : response code = 200")
policyID := strconv.Itoa(addPolicyID)
httpStatusCode, err = apiTest.GetPolicyByID(*admin, policyID)
if err != nil {
t.Error("Error while get policy", err.Error())
t.Log(err)
} else {
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
func TestRepPolicyAPIPut(t *testing.T) {
cases := []*codeCheckingCase{
// 404
&codeCheckingCase{
request: &testingRequest{
method: http.MethodPut,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, 10000),
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 400, invalid trigger
&codeCheckingCase{
request: &testingRequest{
method: http.MethodPut,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID),
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
&models.Project{
ProjectID: projectID,
},
},
Targets: []*models.RepTarget{
&models.RepTarget{
ID: targetID,
},
},
Filters: []rep_models.FilterItem{
rep_models.FilterItem{
Kind: replication.FilterItemKindRepository,
Value: "*",
},
},
Trigger: &rep_models.Trigger{
Kind: "invalid_trigger_kind",
},
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
},
// 200
&codeCheckingCase{
request: &testingRequest{
method: http.MethodPut,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID),
bodyJSON: &api_models.ReplicationPolicy{
Name: policyName,
Projects: []*models.Project{
&models.Project{
ProjectID: projectID,
},
},
Targets: []*models.RepTarget{
&models.RepTarget{
ID: targetID,
},
},
Filters: []rep_models.FilterItem{
rep_models.FilterItem{
Kind: replication.FilterItemKindRepository,
Value: "*",
},
},
Trigger: &rep_models.Trigger{
Kind: replication.TriggerKindImmediately,
},
},
credential: sysAdmin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}
func TestPolicyUpdateInfo(t *testing.T) {
var httpStatusCode int
var err error
targetID := int64(CommonGetTarget())
policyInfo := &apilib.RepPolicyUpdate{TargetId: targetID, Name: "testNewName"}
assert := assert.New(t)
apiTest := newHarborAPI()
fmt.Println("Testing Policy PUT API to update policyInfo")
//-------------------case 1 : response code = 200------------------------//
fmt.Println("case 1 : response code = 200")
policyID := strconv.Itoa(addPolicyID)
httpStatusCode, err = apiTest.PutPolicyInfoByID(*admin, policyID, *policyInfo)
if err != nil {
t.Error("Error while update policyInfo", err.Error())
t.Log(err)
} else {
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
func TestRepPolicyAPIDelete(t *testing.T) {
cases := []*codeCheckingCase{
// 404
&codeCheckingCase{
request: &testingRequest{
method: http.MethodDelete,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, 10000),
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 200
&codeCheckingCase{
request: &testingRequest{
method: http.MethodDelete,
url: fmt.Sprintf("%s/%d", repPolicyAPIBasePath, policyID),
credential: sysAdmin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}
func TestPolicyUpdateEnablement(t *testing.T) {
var httpStatusCode int
var err error
enablement := &apilib.RepPolicyEnablementReq{int32(0)}
assert := assert.New(t)
apiTest := newHarborAPI()
fmt.Println("Testing Policy PUT API to update policy enablement")
//-------------------case 1 : response code = 200------------------------//
fmt.Println("case 1 : response code = 200")
policyID := strconv.Itoa(addPolicyID)
httpStatusCode, err = apiTest.PutPolicyEnableByID(*admin, policyID, *enablement)
if err != nil {
t.Error("Error while put policy enablement", err.Error())
t.Log(err)
} else {
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
}
//-------------------case 2 : response code = 404------------------------//
fmt.Println("case 2 : response code = 404,Not Found")
policyID = "111"
httpStatusCode, err = apiTest.PutPolicyEnableByID(*admin, policyID, *enablement)
if err != nil {
t.Error("Error while put policy enablement", err.Error())
t.Log(err)
} else {
assert.Equal(int(404), httpStatusCode, "httpStatusCode should be 404")
func TestConvertToRepPolicy(t *testing.T) {
cases := []struct {
input *api_models.ReplicationPolicy
expected rep_models.ReplicationPolicy
}{
{
input: nil,
expected: rep_models.ReplicationPolicy{},
},
{
input: &api_models.ReplicationPolicy{
ID: 1,
Name: "policy",
Description: "description",
Filters: []rep_models.FilterItem{
rep_models.FilterItem{
Kind: "filter_kind_01",
Value: "*",
},
},
ReplicateDeletion: true,
Trigger: &rep_models.Trigger{
Kind: "trigger_kind_01",
Param: "{param}",
},
Projects: []*models.Project{
&models.Project{
ProjectID: 1,
},
},
Targets: []*models.RepTarget{
&models.RepTarget{
ID: 1,
},
},
},
expected: rep_models.ReplicationPolicy{
ID: 1,
Name: "policy",
Description: "description",
Filters: []rep_models.FilterItem{
rep_models.FilterItem{
Kind: "filter_kind_01",
Value: "*",
},
},
ReplicateDeletion: true,
Trigger: &rep_models.Trigger{
Kind: "trigger_kind_01",
Param: "{param}",
},
ProjectIDs: []int64{1},
TargetIDs: []int64{1},
},
},
}
}
func TestPolicyDelete(t *testing.T) {
var httpStatusCode int
var err error
assert := assert.New(t)
apiTest := newHarborAPI()
fmt.Println("Testing Policy Delete API")
//-------------------case 1 : response code = 412------------------------//
fmt.Println("case 1 : response code = 412:policy is enabled, can not be deleted")
CommonPolicyEabled(addPolicyID, 1)
policyID := strconv.Itoa(addPolicyID)
httpStatusCode, err = apiTest.DeletePolicyByID(*admin, policyID)
if err != nil {
t.Error("Error while delete policy", err.Error())
t.Log(err)
} else {
assert.Equal(int(412), httpStatusCode, "httpStatusCode should be 412")
}
//-------------------case 2 : response code = 200------------------------//
fmt.Println("case 2 : response code = 200")
CommonPolicyEabled(addPolicyID, 0)
policyID = strconv.Itoa(addPolicyID)
httpStatusCode, err = apiTest.DeletePolicyByID(*admin, policyID)
if err != nil {
t.Error("Error while delete policy", err.Error())
t.Log(err)
} else {
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
}
CommonDelTarget()
for _, c := range cases {
assert.EqualValues(t, c.expected, convertToRepPolicy(c.input))
}
}

1
src/ui/router.go
View File

@ -108,7 +108,6 @@ func initRouters() {
beego.Router("/api/policies/replication/:id([0-9]+)", &api.RepPolicyAPI{})
beego.Router("/api/policies/replication", &api.RepPolicyAPI{}, "get:List")
beego.Router("/api/policies/replication", &api.RepPolicyAPI{}, "post:Post")
beego.Router("/api/policies/replication/:id([0-9]+)/enablement", &api.RepPolicyAPI{}, "put:UpdateEnablement")
beego.Router("/api/targets/", &api.TargetAPI{}, "get:List")
beego.Router("/api/targets/", &api.TargetAPI{}, "post:Post")
beego.Router("/api/targets/:id([0-9]+)", &api.TargetAPI{})

10
tests/apitests/apilib/rep_policy_post.go
View File

@ -22,10 +22,6 @@
package apilib
import (
"github.com/vmware/harbor/src/common/models"
)
type RepPolicyPost struct {
// The project ID.
@ -36,10 +32,4 @@ type RepPolicyPost struct {
// The policy name.
Name string `json:"name,omitempty"`
// Trigger
Trigger *models.RepTrigger `json:"trigger"`
// Filters
Filters []*models.RepFilter `json:"filters"`
}