add ut for tag retention controller

Signed-off-by: Ziming Zhang <zziming@vmware.com>
Change-Id: I1469ee13675537ec389a068e4bc29e457b402fa4
This commit is contained in:
Ziming Zhang 2019-07-25 15:39:20 +08:00
parent 3a7066346b
commit 83b045f5ec
10 changed files with 590 additions and 43 deletions

View File

@ -118,12 +118,13 @@ func Init() error {
retentionLauncher = retention.NewLauncher(projectMgr, repositoryMgr, retentionMgr)
retentionController = retention.NewAPIController(projectMgr, repositoryMgr, retentionScheduler, retentionLauncher)
retentionController = retention.NewAPIController(retentionMgr, projectMgr, repositoryMgr, retentionScheduler, retentionLauncher)
callbackFun := func(p interface{}) error {
r, ok := p.(retention.TriggerParam)
if ok {
return retentionController.TriggerRetentionExec(r.PolicyID, r.Trigger, false)
_, err := retentionController.TriggerRetentionExec(r.PolicyID, r.Trigger, false)
return err
}
return errors.New("bad retention callback param")
}

View File

@ -160,6 +160,16 @@ func init() {
beego.Router("/api/replication/policies", &ReplicationPolicyAPI{}, "get:List;post:Create")
beego.Router("/api/replication/policies/:id([0-9]+)", &ReplicationPolicyAPI{}, "get:Get;put:Update;delete:Delete")
beego.Router("/api/retentions/metadatas", &RetentionAPI{}, "get:GetMetadatas")
beego.Router("/api/retentions/:id", &RetentionAPI{}, "get:GetRetention")
beego.Router("/api/retentions", &RetentionAPI{}, "post:CreateRetention")
beego.Router("/api/retentions/:id", &RetentionAPI{}, "put:UpdateRetention")
beego.Router("/api/retentions/:id/executions", &RetentionAPI{}, "post:TriggerRetentionExec")
beego.Router("/api/retentions/:id/executions/:eid", &RetentionAPI{}, "patch:OperateRetentionExec")
beego.Router("/api/retentions/:id/executions", &RetentionAPI{}, "get:ListRetentionExecs")
beego.Router("/api/retentions/:id/executions/:eid/tasks", &RetentionAPI{}, "get:ListRetentionExecTasks")
beego.Router("/api/retentions/:id/executions/:eid/tasks/:tid", &RetentionAPI{}, "get:GetRetentionExecTaskLog")
// Charts are controlled under projects
chartRepositoryAPIType := &ChartRepositoryAPI{}
beego.Router("/api/chartrepo/health", chartRepositoryAPIType, "get:GetHealthStatus")

View File

@ -131,7 +131,10 @@ func (r *RetentionAPI) GetMetadatas() {
]
}
`
r.WriteJSONData(data)
w := r.Ctx.ResponseWriter
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
w.Write([]byte(data))
}
// GetRetention Get Retention
@ -166,7 +169,7 @@ func (r *RetentionAPI) CreateRetention() {
switch p.Scope.Level {
case policy.ScopeLevelProject:
if p.Scope.Reference <= 0 {
r.SendBadRequestError(fmt.Errorf("Invalid Project id %d", p.Scope.Reference))
r.SendBadRequestError(fmt.Errorf("invalid Project id %d", p.Scope.Reference))
return
}
@ -175,19 +178,22 @@ func (r *RetentionAPI) CreateRetention() {
r.SendBadRequestError(err)
}
if proj == nil {
r.SendBadRequestError(fmt.Errorf("Invalid Project id %d", p.Scope.Reference))
r.SendBadRequestError(fmt.Errorf("invalid Project id %d", p.Scope.Reference))
}
default:
r.SendBadRequestError(fmt.Errorf("scope %s is not support", p.Scope.Level))
return
}
if err = retentionController.CreateRetention(p); err != nil {
id, err := retentionController.CreateRetention(p)
if err != nil {
r.SendInternalServerError(err)
return
}
if err := r.pm.GetMetadataManager().Add(p.Scope.Reference,
map[string]string{"retention_id": strconv.FormatInt(p.Scope.Reference, 10)}); err != nil {
r.SendInternalServerError(err)
}
r.Redirect(http.StatusCreated, strconv.FormatInt(id, 10))
}
// UpdateRetention Update Retention
@ -238,10 +244,12 @@ func (r *RetentionAPI) TriggerRetentionExec() {
if !r.requireAccess(p, rbac.ActionUpdate) {
return
}
if err = retentionController.TriggerRetentionExec(id, retention.ExecutionTriggerManual, d.DryRun); err != nil {
eid, err := retentionController.TriggerRetentionExec(id, retention.ExecutionTriggerManual, d.DryRun)
if err != nil {
r.SendInternalServerError(err)
return
}
r.Redirect(http.StatusCreated, strconv.FormatInt(eid, 10))
}
// OperateRetentionExec Operate Retention Execution

View File

@ -0,0 +1,294 @@
package api
import (
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/pkg/retention/dao"
"github.com/goharbor/harbor/src/pkg/retention/dao/models"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
"github.com/stretchr/testify/require"
"net/http"
"testing"
"time"
)
func TestGetMetadatas(t *testing.T) {
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/retentions/metadatas",
credential: sysAdmin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}
func TestCreatePolicy(t *testing.T) {
p1 := &policy.Metadata{
Algorithm: "or",
Rules: []rule.Metadata{
{
ID: 1,
Priority: 1,
Template: "recentXdays",
Parameters: rule.Parameters{
"num": 10,
},
TagSelectors: []*rule.Selector{
{
Kind: "label",
Decoration: "with",
Pattern: "latest",
},
{
Kind: "regularExpression",
Decoration: "matches",
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "regularExpression",
Decoration: "matches",
Pattern: ".+",
},
},
},
},
},
Trigger: &policy.Trigger{
Kind: "Schedule",
Settings: map[string]interface{}{
"cron": "* 22 11 * * *",
},
},
Scope: &policy.Scope{
Level: "project",
Reference: 1,
},
}
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodPost,
url: "/api/retentions",
},
code: http.StatusUnauthorized,
},
{
request: &testingRequest{
method: http.MethodPost,
url: "/api/retentions",
bodyJSON: p1,
credential: sysAdmin,
},
code: http.StatusOK,
},
{
request: &testingRequest{
method: http.MethodPost,
url: "/api/retentions",
bodyJSON: &policy.Metadata{
Algorithm: "NODEF",
Rules: []rule.Metadata{
{
ID: 1,
Priority: 1,
Template: "recentXdays",
Parameters: rule.Parameters{
"num": 10,
},
TagSelectors: []*rule.Selector{
{
Kind: "label",
Decoration: "with",
Pattern: "latest",
},
{
Kind: "regularExpression",
Decoration: "matches",
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "regularExpression",
Decoration: "matches",
Pattern: ".+",
},
},
},
},
},
Trigger: &policy.Trigger{
Kind: "Schedule",
Settings: map[string]interface{}{},
},
Scope: &policy.Scope{
Level: "project",
Reference: 1,
},
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
},
}
runCodeCheckingCases(t, cases...)
}
func TestPolicy(t *testing.T) {
p := &policy.Metadata{
Algorithm: "or",
Rules: []rule.Metadata{
{
ID: 1,
Priority: 1,
Template: "recentXdays",
Parameters: rule.Parameters{
"num": 10,
},
TagSelectors: []*rule.Selector{
{
Kind: "label",
Decoration: "with",
Pattern: "latest",
},
{
Kind: "regularExpression",
Decoration: "matches",
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "regularExpression",
Decoration: "matches",
Pattern: ".+",
},
},
},
},
},
Trigger: &policy.Trigger{
Kind: "Schedule",
Settings: map[string]interface{}{
"cron": "* 22 11 * * *",
},
},
Scope: &policy.Scope{
Level: "project",
Reference: 1,
},
}
p1 := &models.RetentionPolicy{
ScopeLevel: p.Scope.Level,
TriggerKind: p.Trigger.Kind,
CreateTime: time.Now(),
UpdateTime: time.Now(),
}
data, _ := json.Marshal(p)
p1.Data = string(data)
id, err := dao.CreatePolicy(p1)
require.Nil(t, err)
require.True(t, id > 0)
cases := []*codeCheckingCase{
{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("/api/retentions/%d", id),
credential: sysAdmin,
},
code: http.StatusOK,
},
{
request: &testingRequest{
method: http.MethodPut,
url: fmt.Sprintf("/api/retentions/%d", id),
bodyJSON: &policy.Metadata{
Algorithm: "or",
Rules: []rule.Metadata{
{
ID: 1,
Priority: 1,
Template: "recentXdays",
Parameters: rule.Parameters{
"num": 10,
},
TagSelectors: []*rule.Selector{
{
Kind: "label",
Decoration: "with",
Pattern: "latest",
},
{
Kind: "regularExpression",
Decoration: "matches",
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "regularExpression",
Decoration: "matches",
Pattern: "b.+",
},
},
},
},
},
Trigger: &policy.Trigger{
Kind: "Schedule",
Settings: map[string]interface{}{
"cron": "* 22 11 * * *",
},
},
Scope: &policy.Scope{
Level: "project",
Reference: 1,
},
},
credential: sysAdmin,
},
code: http.StatusOK,
},
{
request: &testingRequest{
method: http.MethodPost,
url: fmt.Sprintf("/api/retentions/%d/executions", id),
bodyJSON: &struct {
DryRun bool `json:"dry_run"`
}{
DryRun: false,
},
credential: sysAdmin,
},
code: http.StatusOK,
},
{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("/api/retentions/%d/executions", id),
credential: sysAdmin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}

View File

@ -40,16 +40,18 @@ type APIController interface {
GetRetention(id int64) (*policy.Metadata, error)
CreateRetention(p *policy.Metadata) error
CreateRetention(p *policy.Metadata) (int64, error)
UpdateRetention(p *policy.Metadata) error
DeleteRetention(id int64) error
TriggerRetentionExec(policyID int64, trigger string, dryRun bool) error
TriggerRetentionExec(policyID int64, trigger string, dryRun bool) (int64, error)
OperateRetentionExec(eid int64, action string) error
GetRetentionExec(eid int64) (*Execution, error)
ListRetentionExecs(policyID int64, query *q.Query) ([]*Execution, error)
ListRetentionExecTasks(executionID int64, query *q.Query) ([]*Task, error)
@ -83,26 +85,28 @@ func (r *DefaultAPIController) GetRetention(id int64) (*policy.Metadata, error)
}
// CreateRetention Create Retention
func (r *DefaultAPIController) CreateRetention(p *policy.Metadata) error {
func (r *DefaultAPIController) CreateRetention(p *policy.Metadata) (int64, error) {
if p.Trigger.Kind == policy.TriggerKindSchedule {
if p.Trigger.Settings != nil {
cron, ok := p.Trigger.Settings[policy.TriggerSettingsCron]
if ok {
jobid, err := r.scheduler.Schedule(cron.(string), SchedulerCallback, TriggerParam{
PolicyID: p.ID,
Trigger: ExecutionTriggerSchedule,
})
if err != nil {
return err
}
p.Trigger.References[policy.TriggerReferencesJobid] = jobid
cron, ok := p.Trigger.Settings[policy.TriggerSettingsCron]
if ok {
jobid, err := r.scheduler.Schedule(cron.(string), SchedulerCallback, TriggerParam{
PolicyID: p.ID,
Trigger: ExecutionTriggerSchedule,
})
if err != nil {
return 0, err
}
if p.Trigger.References == nil {
p.Trigger.References = map[string]interface{}{}
}
p.Trigger.References[policy.TriggerReferencesJobid] = jobid
}
}
if _, err := r.manager.CreatePolicy(p); err != nil {
return err
id, err := r.manager.CreatePolicy(p)
if err != nil {
return 0, err
}
return nil
return id, nil
}
// UpdateRetention Update Retention
@ -143,7 +147,7 @@ func (r *DefaultAPIController) UpdateRetention(p *policy.Metadata) error {
}
}
if needUn {
err = r.scheduler.UnSchedule(p0.Trigger.References[policy.TriggerReferencesJobid].(int64))
err = r.scheduler.UnSchedule((p0.Trigger.References[policy.TriggerReferencesJobid].(int64)))
if err != nil {
return err
}
@ -179,10 +183,10 @@ func (r *DefaultAPIController) DeleteRetention(id int64) error {
}
// TriggerRetentionExec Trigger Retention Execution
func (r *DefaultAPIController) TriggerRetentionExec(policyID int64, trigger string, dryRun bool) error {
func (r *DefaultAPIController) TriggerRetentionExec(policyID int64, trigger string, dryRun bool) (int64, error) {
p, err := r.manager.GetPolicy(policyID)
if err != nil {
return err
return 0, err
}
exec := &Execution{
@ -195,7 +199,7 @@ func (r *DefaultAPIController) TriggerRetentionExec(policyID int64, trigger stri
id, err := r.manager.CreateExecution(exec)
num, err := r.launcher.Launch(p, id, dryRun)
if err != nil {
return err
return 0, err
}
if num == 0 {
exec := &Execution{
@ -205,10 +209,10 @@ func (r *DefaultAPIController) TriggerRetentionExec(policyID int64, trigger stri
}
err = r.manager.UpdateExecution(exec)
if err != nil {
return err
return 0, err
}
}
return err
return id, err
}
@ -229,6 +233,11 @@ func (r *DefaultAPIController) OperateRetentionExec(eid int64, action string) er
}
}
// GetRetentionExec Get Retention Execution
func (r *DefaultAPIController) GetRetentionExec(executionID int64) (*Execution, error) {
return r.manager.GetExecution(executionID)
}
// ListRetentionExecs List Retention Executions
func (r *DefaultAPIController) ListRetentionExecs(policyID int64, query *q.Query) ([]*Execution, error) {
return r.manager.ListExecutions(policyID, query)
@ -238,8 +247,10 @@ func (r *DefaultAPIController) ListRetentionExecs(policyID int64, query *q.Query
func (r *DefaultAPIController) ListRetentionExecTasks(executionID int64, query *q.Query) ([]*Task, error) {
q1 := &q.TaskQuery{
ExecutionID: executionID,
PageNumber: query.PageNumber,
PageSize: query.PageSize,
}
if query != nil {
q1.PageSize = query.PageSize
q1.PageNumber = query.PageNumber
}
return r.manager.ListTasks(q1)
}
@ -255,9 +266,9 @@ func (r *DefaultAPIController) HandleHook(policyID string, event *job.StatusChan
}
// NewAPIController ...
func NewAPIController(projectManager project.Manager, repositoryMgr repository.Manager, scheduler scheduler.Scheduler, retentionLauncher Launcher) APIController {
func NewAPIController(retentionMgr Manager, projectManager project.Manager, repositoryMgr repository.Manager, scheduler scheduler.Scheduler, retentionLauncher Launcher) APIController {
return &DefaultAPIController{
manager: NewManager(),
manager: retentionMgr,
launcher: retentionLauncher,
projectManager: projectManager,
repositoryMgr: repositoryMgr,

View File

@ -0,0 +1,203 @@
package retention
import (
"github.com/goharbor/harbor/src/pkg/retention/dep"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
"github.com/stretchr/testify/suite"
"testing"
)
type ControllerTestSuite struct {
suite.Suite
oldClient dep.Client
}
// SetupSuite ...
func (s *ControllerTestSuite) SetupSuite() {
}
// TestController ...
func TestController(t *testing.T) {
suite.Run(t, new(ControllerTestSuite))
}
func (s *ControllerTestSuite) TestPolicy() {
projectMgr := &fakeProjectManager{}
repositoryMgr := &fakeRepositoryManager{}
retentionScheduler := &fakeRetentionScheduler{}
retentionLauncher := &fakeLauncher{}
retentionMgr := NewManager()
c := NewAPIController(retentionMgr, projectMgr, repositoryMgr, retentionScheduler, retentionLauncher)
p1 := &policy.Metadata{
Algorithm: "or",
Rules: []rule.Metadata{
{
ID: 1,
Priority: 1,
Template: "recentXdays",
Parameters: rule.Parameters{
"num": 10,
},
TagSelectors: []*rule.Selector{
{
Kind: "label",
Decoration: "with",
Pattern: "latest",
},
{
Kind: "regularExpression",
Decoration: "matches",
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "regularExpression",
Decoration: "matches",
Pattern: ".+",
},
},
},
},
},
Trigger: &policy.Trigger{
Kind: "Schedule",
Settings: map[string]interface{}{
"cron": "* 22 11 * * *",
},
},
Scope: &policy.Scope{
Level: "project",
Reference: 1,
},
}
id, err := c.CreateRetention(p1)
s.Require().Nil(err)
s.Require().True(id > 0)
p1, err = c.GetRetention(id)
s.Require().Nil(err)
s.Require().EqualValues("project", p1.Scope.Level)
s.Require().True(p1.ID > 0)
p1.Scope.Level = "test"
err = c.UpdateRetention(p1)
s.Require().Nil(err)
p1, err = c.GetRetention(id)
s.Require().Nil(err)
s.Require().EqualValues("test", p1.Scope.Level)
err = c.DeleteRetention(id)
s.Require().Nil(err)
p1, err = c.GetRetention(id)
s.Require().Nil(err)
s.Require().Nil(p1)
}
func (s *ControllerTestSuite) TestExecution() {
projectMgr := &fakeProjectManager{}
repositoryMgr := &fakeRepositoryManager{}
retentionScheduler := &fakeRetentionScheduler{}
retentionLauncher := &fakeLauncher{}
retentionMgr := NewManager()
m := NewAPIController(retentionMgr, projectMgr, repositoryMgr, retentionScheduler, retentionLauncher)
p1 := &policy.Metadata{
Algorithm: "or",
Rules: []rule.Metadata{
{
ID: 1,
Priority: 1,
Template: "recentXdays",
Parameters: rule.Parameters{
"num": 10,
},
TagSelectors: []*rule.Selector{
{
Kind: "label",
Decoration: "with",
Pattern: "latest",
},
{
Kind: "regularExpression",
Decoration: "matches",
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "regularExpression",
Decoration: "matches",
Pattern: ".+",
},
},
},
},
},
Trigger: &policy.Trigger{
Kind: "Schedule",
Settings: map[string]interface{}{
"cron": "* 22 11 * * *",
},
},
Scope: &policy.Scope{
Level: "project",
Reference: 1,
},
}
policyID, err := m.CreateRetention(p1)
s.Require().Nil(err)
s.Require().True(policyID > 0)
id, err := m.TriggerRetentionExec(policyID, ExecutionTriggerManual, false)
s.Require().Nil(err)
s.Require().True(id > 0)
e1, err := m.GetRetentionExec(id)
s.Require().Nil(err)
s.Require().NotNil(e1)
s.Require().EqualValues(id, e1.ID)
err = m.OperateRetentionExec(id, "stop")
s.Require().Nil(err)
es, err := m.ListRetentionExecs(policyID, nil)
s.Require().Nil(err)
s.Require().EqualValues(1, len(es))
ts, err := m.ListRetentionExecTasks(id, nil)
s.Require().Nil(err)
s.Require().EqualValues(0, len(ts))
}
type fakeRetentionScheduler struct {
}
func (f *fakeRetentionScheduler) Schedule(cron string, callbackFuncName string, params interface{}) (int64, error) {
return 111, nil
}
func (f *fakeRetentionScheduler) UnSchedule(id int64) error {
return nil
}
type fakeLauncher struct {
}
func (f *fakeLauncher) Stop(executionID int64) error {
return nil
}
func (f *fakeLauncher) Launch(policy *policy.Metadata, executionID int64, isDryRun bool) (int64, error) {
return 0, nil
}

View File

@ -23,7 +23,7 @@ func TestMain(m *testing.M) {
func TestPolicy(t *testing.T) {
p := &policy.Metadata{
Algorithm: "OR",
Algorithm: "or",
Rules: []rule.Metadata{
{
ID: 1,
@ -101,7 +101,7 @@ func TestPolicy(t *testing.T) {
func TestExecution(t *testing.T) {
p := &policy.Metadata{
Algorithm: "OR",
Algorithm: "or",
Rules: []rule.Metadata{
{
ID: 1,

View File

@ -109,6 +109,11 @@ func (d *DefaultManager) GetPolicy(id int64) (*policy.Metadata, error) {
return nil, err
}
p.ID = id
if p.Trigger.Settings != nil {
if _, ok := p.Trigger.References[policy.TriggerReferencesJobid]; ok {
p.Trigger.References[policy.TriggerReferencesJobid] = int64(p.Trigger.References[policy.TriggerReferencesJobid].(float64))
}
}
return p, nil
}
@ -118,8 +123,8 @@ func (d *DefaultManager) CreateExecution(execution *Execution) (int64, error) {
exec.PolicyID = execution.PolicyID
exec.StartTime = time.Now()
exec.DryRun = execution.DryRun
exec.Status = "Running"
exec.Trigger = "manual"
exec.Status = execution.Status
exec.Trigger = execution.Trigger
return dao.CreateExecution(exec)
}

View File

@ -23,7 +23,7 @@ func TestMain(m *testing.M) {
func TestPolicy(t *testing.T) {
m := NewManager()
p1 := &policy.Metadata{
Algorithm: "OR",
Algorithm: "or",
Rules: []rule.Metadata{
{
ID: 1,
@ -94,7 +94,7 @@ func TestPolicy(t *testing.T) {
func TestExecution(t *testing.T) {
m := NewManager()
p1 := &policy.Metadata{
Algorithm: "OR",
Algorithm: "or",
Rules: []rule.Metadata{
{
ID: 1,

View File

@ -15,6 +15,7 @@
package policy
import (
"github.com/astaxie/beego/validation"
"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
)
@ -41,7 +42,7 @@ type Metadata struct {
// Algorithm applied to the rules
// "OR" / "AND"
Algorithm string `json:"algorithm" valid:"Required;Match(/^(OR|AND)$/)"`
Algorithm string `json:"algorithm" valid:"Required;Match(or)"`
// Rule collection
Rules []rule.Metadata `json:"rules"`
@ -56,15 +57,29 @@ type Metadata struct {
Capacity int `json:"cap"`
}
// Valid Valid
func (m *Metadata) Valid(v *validation.Validation) {
if m.Trigger.Kind == TriggerKindSchedule {
if m.Trigger.Settings == nil {
_ = v.SetError("Trigger.Settings", "Trigger.Settings is required")
} else {
if _, ok := m.Trigger.Settings[TriggerSettingsCron]; !ok {
_ = v.SetError("Trigger.Settings", "cron in Trigger.Settings is required")
}
}
}
}
// Trigger of the policy
type Trigger struct {
// Const string to declare the trigger type
// 'Schedule'
Kind string `json:"kind"`
Kind string `json:"kind" valid:"Required"`
// Settings for the specified trigger
// '[cron]="* 22 11 * * *"' for the 'Schedule'
Settings map[string]interface{} `json:"settings"`
Settings map[string]interface{} `json:"settings" valid:"Required"`
// References of the trigger
// e.g: schedule job ID