From 2c2025102b4ec5f98948c022afd5d004412ffee3 Mon Sep 17 00:00:00 2001 From: Ziming Zhang Date: Tue, 9 Jul 2019 15:29:19 +0800 Subject: [PATCH] retention api Change-Id: I70f2c34d6bb96ecf4cb5359e2b1ab2dbb99fdbf9 --- src/core/router.go | 11 ++ src/pkg/retention/controllers/retention.go | 74 ++++++++++++++ src/pkg/retention/dao/models/retention.go | 64 ++++++++++++ src/pkg/retention/dao/retention.go | 37 +++++++ src/pkg/retention/dao/retention_test.go | 96 ++++++++++++++++++ src/pkg/retention/manager.go | 112 ++++++++++++++++++++- src/pkg/retention/models.go | 6 +- src/pkg/retention/policy/models.go | 2 +- src/pkg/retention/policy/rule/models.go | 2 +- 9 files changed, 394 insertions(+), 10 deletions(-) create mode 100644 src/pkg/retention/controllers/retention.go create mode 100644 src/pkg/retention/dao/models/retention.go create mode 100644 src/pkg/retention/dao/retention.go create mode 100644 src/pkg/retention/dao/retention_test.go diff --git a/src/core/router.go b/src/core/router.go index 344967c1d..8170129d5 100644 --- a/src/core/router.go +++ b/src/core/router.go @@ -24,6 +24,7 @@ import ( "github.com/goharbor/harbor/src/core/service/notifications/jobs" "github.com/goharbor/harbor/src/core/service/notifications/registry" "github.com/goharbor/harbor/src/core/service/token" + retentionCtl "github.com/goharbor/harbor/src/pkg/retention/controllers" "github.com/astaxie/beego" ) @@ -139,6 +140,16 @@ func initRouters() { beego.Router("/api/registries/:id/info", &api.RegistryAPI{}, "get:GetInfo") beego.Router("/api/registries/:id/namespace", &api.RegistryAPI{}, "get:GetNamespace") + beego.Router("/api/retentions/:id", &retentionCtl.RetentionAPI{}, "get:GetRetention") + beego.Router("/api/retentions", &retentionCtl.RetentionAPI{}, "post:CreateRetention") + beego.Router("/api/retentions/:id", &retentionCtl.RetentionAPI{}, "put:UpdateRetention") + beego.Router("/api/retentions/:id", &retentionCtl.RetentionAPI{}, "delete:DeleteRetention") + beego.Router("/api/retentions/:id/executions", &retentionCtl.RetentionAPI{}, "delete:TriggerRetentionExec") + beego.Router("/api/retentions/:id/executions/:eid", &retentionCtl.RetentionAPI{}, "delete:OperateRetentionExec") + beego.Router("/api/retentions/:id/executions/:eid", &retentionCtl.RetentionAPI{}, "get:GetRetentionExec") + beego.Router("/api/retentions/:id/executions", &retentionCtl.RetentionAPI{}, "get:ListRetentionExec") + beego.Router("/api/retentions/:id/executions/:eid/histories", &retentionCtl.RetentionAPI{}, "get:ListRetentionExecHistory") + beego.Router("/v2/*", &controllers.RegistryProxy{}, "*:Handle") // APIs for chart repository diff --git a/src/pkg/retention/controllers/retention.go b/src/pkg/retention/controllers/retention.go new file mode 100644 index 000000000..7f4eadc0a --- /dev/null +++ b/src/pkg/retention/controllers/retention.go @@ -0,0 +1,74 @@ +package controllers + +import ( + "github.com/goharbor/harbor/src/core/api" + "github.com/goharbor/harbor/src/pkg/retention" +) + +type RetentionAPI struct { + api.BaseController + manager retention.Manager +} + +// Prepare validates the user +func (t *RetentionAPI) Prepare() { + t.BaseController.Prepare() + t.manager = retention.NewManager() +} + +func (r *RetentionAPI) GetRetention() { + id, err := r.GetIDFromURL() + if err != nil { + r.SendBadRequestError(err) + return + } + p, err := r.manager.GetPolicy(id) + if err != nil { + r.SendBadRequestError(err) + return + } + r.Data["json"] = p + r.ServeJSON() +} + +func (r *RetentionAPI) CreateRetention() { + r.manager.CreatePolicy(nil) +} + +func (r *RetentionAPI) UpdateRetention() { + _, err := r.GetIDFromURL() + if err != nil { + r.SendBadRequestError(err) + return + } + r.manager.UpdatePolicy(nil) +} + +func (r *RetentionAPI) DeleteRetention() { + id, err := r.GetIDFromURL() + if err != nil { + r.SendBadRequestError(err) + return + } + r.manager.DeletePolicy(id) +} + +func (r *RetentionAPI) TriggerRetentionExec() { + //r.manager.CreateExecution(nil) +} + +func (r *RetentionAPI) OperateRetentionExec() { + r.manager.UpdateExecution(nil) +} + +func (r *RetentionAPI) GetRetentionExec() { + //r.manager.GetExecution(eid) +} + +func (r *RetentionAPI) ListRetentionExec() { + r.manager.ListExecutions(nil) +} + +func (r *RetentionAPI) ListRetentionExecHistory() { + //r.manager.ListHistories(eid, nil) +} diff --git a/src/pkg/retention/dao/models/retention.go b/src/pkg/retention/dao/models/retention.go new file mode 100644 index 000000000..c8ad47b5e --- /dev/null +++ b/src/pkg/retention/dao/models/retention.go @@ -0,0 +1,64 @@ +package models + +import ( + "github.com/astaxie/beego/orm" + "time" +) + +func init() { + orm.RegisterModel( + new (RetentionPolicy), + new (RetentionExecution), + new (RetentionTask), + new (RetentionScheduleJob), + ) +} + +type RetentionPolicy struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + // 'system', 'project' and 'repository' + ScopeLevel string + ScopeReference int64 + TriggerKind string + // json format, include algorithm, rules, exclusions + Data string + CreateTime time.Time + UpdateTime time.Time +} + +type RetentionExecution struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + PolicyID int64 + Status string + StatusText string + Dry bool + // manual, scheduled + Trigger string + Total int + Succeed int + Failed int + InProgress int + Stopped int + StartTime time.Time + EndTime time.Time +} + +type RetentionTask struct { + ID int64 + ExecutionID int64 + // image, chart + ResourceType string + Resource string + Status string + StartTime time.Time + EndTime time.Time +} + +type RetentionScheduleJob struct { + ID int64 + Status string + PolicyID int64 + JobID int64 + CreateTime time.Time + UpdateTime time.Time +} diff --git a/src/pkg/retention/dao/retention.go b/src/pkg/retention/dao/retention.go new file mode 100644 index 000000000..8630da3cd --- /dev/null +++ b/src/pkg/retention/dao/retention.go @@ -0,0 +1,37 @@ +package dao + +import ( + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/pkg/retention/dao/models" +) + +func CreatePolicy(p *models.RetentionPolicy) (int64, error) { + o := dao.GetOrmer() + return o.Insert(p) +} + +func UpdatePolicy(p *models.RetentionPolicy) error { + o := dao.GetOrmer() + _, err := o.Update(p) + return err +} + +func DeletePolicy(id int64) error { + o := dao.GetOrmer() + _, err := o.Delete(&models.RetentionPolicy{ + ID: id, + }) + return err +} + +func GetPolicy(id int64) (*models.RetentionPolicy, error) { + o := dao.GetOrmer() + p := &models.RetentionPolicy{ + ID: id, + } + if err := o.Read(p); err != nil { + return nil, err + } else { + return p, nil + } +} diff --git a/src/pkg/retention/dao/retention_test.go b/src/pkg/retention/dao/retention_test.go new file mode 100644 index 000000000..bda3bd602 --- /dev/null +++ b/src/pkg/retention/dao/retention_test.go @@ -0,0 +1,96 @@ +package dao + +import ( + "encoding/json" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/pkg/retention/policy" + "github.com/goharbor/harbor/src/pkg/retention/policy/rule" + "github.com/stretchr/testify/assert" + "os" + "strings" + "testing" + "time" + + "github.com/goharbor/harbor/src/pkg/retention/dao/models" +) + +func TestMain(m *testing.M) { + dao.PrepareTestForPostgresSQL() + os.Exit(m.Run()) +} + +func TestPolicy(t *testing.T) { + p := &policy.Metadata{ + ID: 1, + Algorithm: "OR", + Rules: []rule.Metadata{ + { + ID: 1, + Priority: 1, + Template: "recentXdays", + Parameters: rule.Parameters{ + "num": 10, + }, + TagSelectors: []*rule.Selector{ + { + Kind: "label", + Decoration: "with", + Pattern: "latest", + }, + { + Kind: "regularExpression", + Decoration: "matches", + Pattern: "release-[\\d\\.]+", + }, + }, + ScopeSelectors: []*rule.Selector{ + { + Kind: "regularExpression", + Decoration: "matches", + Pattern: ".+", + }, + }, + }, + }, + Trigger: &policy.Trigger{ + Kind: "Schedule", + Settings: map[string]interface{}{ + "cron": "* 22 11 * * *", + }, + }, + Scope: &policy.Scope{ + Level: "project", + Reference: 1, + }, + } + p1 := &models.RetentionPolicy{ + ScopeLevel: p.Scope.Level, + TriggerKind: p.Trigger.Kind, + CreateTime: time.Now(), + UpdateTime: time.Now(), + } + data, _ := json.Marshal(p) + p1.Data = string(data) + + id, err := CreatePolicy(p1) + assert.Nil(t, err) + assert.True(t, id > 0) + + p1, err = GetPolicy(id) + assert.Nil(t, err) + assert.EqualValues(t, "project", p1.ScopeLevel) + + p1.ScopeLevel = "test" + err = UpdatePolicy(p1) + assert.Nil(t, err) + p1, err = GetPolicy(id) + assert.Nil(t, err) + assert.EqualValues(t, "test", p1.ScopeLevel) + + err = DeletePolicy(id) + assert.Nil(t, err) + + p1, err = GetPolicy(id) + assert.NotNil(t, err) + assert.True(t, strings.Contains(err.Error(), "no row found")) +} diff --git a/src/pkg/retention/manager.go b/src/pkg/retention/manager.go index 0a562dab0..9470db2b8 100644 --- a/src/pkg/retention/manager.go +++ b/src/pkg/retention/manager.go @@ -15,32 +15,134 @@ package retention import ( + "encoding/json" + "github.com/goharbor/harbor/src/pkg/retention/dao" + "github.com/goharbor/harbor/src/pkg/retention/dao/models" "github.com/goharbor/harbor/src/pkg/retention/policy" "github.com/goharbor/harbor/src/pkg/retention/q" + "time" ) // Manager defines operations of managing policy type Manager interface { // Create new policy and return uuid - CreatePolicy(p *policy.Metadata) (string, error) + CreatePolicy(p *policy.Metadata) (int64, error) // Update the existing policy // Full update UpdatePolicy(p *policy.Metadata) error // Delete the specified policy // No actual use so far - DeletePolicy(ID string) error + DeletePolicy(ID int64) error // Get the specified policy - GetPolicy(ID string) (*policy.Metadata, error) + GetPolicy(ID int64) (*policy.Metadata, error) // Create a new retention execution CreateExecution(execution *Execution) (string, error) // Update the specified execution UpdateExecution(execution *Execution) error // Get the specified execution - GetExecution(eid string) (*Execution, error) + GetExecution(eid int64) (*Execution, error) // List execution histories ListExecutions(query *q.Query) ([]*Execution, error) // Add new history AppendHistory(history *History) error // List all the histories marked by the specified execution - ListHistories(executionID string, query *q.Query) ([]*History, error) + ListHistories(executionID int64, query *q.Query) ([]*History, error) +} + +type DefaultManager struct { +} + +func (d *DefaultManager) CreatePolicy(p *policy.Metadata) (int64, error) { + var p1 *models.RetentionPolicy + p1.ScopeLevel = p.Scope.Level + p1.TriggerKind = p.Trigger.Kind + data, _ := json.Marshal(p) + p1.Data = string(data) + p1.CreateTime = time.Now() + p1.UpdateTime = p1.CreateTime + return dao.CreatePolicy(p1) +} + +func (d *DefaultManager) UpdatePolicy(p *policy.Metadata) error { + var p1 *models.RetentionPolicy + p1.ID = p.ID + p1.ScopeLevel = p.Scope.Level + p1.TriggerKind = p.Trigger.Kind + data, _ := json.Marshal(p) + p1.Data = string(data) + p1.UpdateTime = time.Now() + return dao.UpdatePolicy(p1) +} + +func (d *DefaultManager) DeletePolicy(id int64) error { + return dao.DeletePolicy(id) +} + +func (d *DefaultManager) GetPolicy(id int64) (*policy.Metadata, error) { + if p1,err:=dao.GetPolicy(id);err!=nil{ + return nil,err + }else{ + var p *policy.Metadata + if err=json.Unmarshal([]byte(p1.Data), p);err!=nil{ + return nil,err + }else{ + return p,nil + } + } +} + +func (d *DefaultManager) CreateExecution(execution *Execution) (string, error) { + panic("implement me") +} + +func (d *DefaultManager) UpdateExecution(execution *Execution) error { + panic("implement me") +} + +func (d *DefaultManager) ListExecutions(query *q.Query) ([]*Execution, error) { + return []*Execution{ + { + ID: 1, + PolicyID: 1, + StartTime: time.Now().Add(-time.Minute), + EndTime: time.Now(), + Status: "Success", + }, + { + ID: 2, + PolicyID: 1, + StartTime: time.Now().Add(-time.Minute), + EndTime: time.Now(), + Status: "Failed", + }, + { + ID: 3, + PolicyID: 1, + StartTime: time.Now().Add(-time.Minute), + EndTime: time.Now(), + Status: "Running", + }, + }, nil +} + +func (d *DefaultManager) GetExecution(eid int64) (*Execution, error) { + return &Execution{ + ID: 1, + PolicyID: 1, + StartTime: time.Now().Add(-time.Minute), + EndTime: time.Now(), + Status: "Success", + }, nil +} + +func (d *DefaultManager) ListHistories(executionID int64, query *q.Query) ([]*History, error) { + panic("implement me") +} + +func (d *DefaultManager) AppendHistory(history *History) error { + panic("implement me") +} + +func NewManager() Manager { + return &DefaultManager{} } diff --git a/src/pkg/retention/models.go b/src/pkg/retention/models.go index f2a190500..ba907fe83 100644 --- a/src/pkg/retention/models.go +++ b/src/pkg/retention/models.go @@ -18,8 +18,8 @@ import "time" // Execution of retention type Execution struct { - ID string `json:"id"` - PolicyID string `json:"policy_id"` + ID int `json:"id"` + PolicyID int `json:"policy_id"` StartTime time.Time `json:"start_time"` EndTime time.Time `json:"end_time"` Status string `json:"status"` @@ -37,7 +37,7 @@ type TaskSubmitResult struct { type History struct { ExecutionID string `json:"execution_id"` Rule struct { - ID string `json:"id"` + ID int `json:"id"` DisplayText string `json:"display_text"` } `json:"rule_id"` // full path: :ns/:repo:tag diff --git a/src/pkg/retention/policy/models.go b/src/pkg/retention/policy/models.go index ae0e764f8..e25804240 100644 --- a/src/pkg/retention/policy/models.go +++ b/src/pkg/retention/policy/models.go @@ -26,7 +26,7 @@ const ( // Metadata of policy type Metadata struct { // UUID of the policy - ID string `json:"id"` + ID int64 `json:"-"` // Algorithm applied to the rules // "OR" / "AND" diff --git a/src/pkg/retention/policy/rule/models.go b/src/pkg/retention/policy/rule/models.go index b14841e72..018c3a239 100644 --- a/src/pkg/retention/policy/rule/models.go +++ b/src/pkg/retention/policy/rule/models.go @@ -17,7 +17,7 @@ package rule // Metadata of the retention rule type Metadata struct { // UUID of rule - ID string `json:"id"` + ID int `json:"id"` // Priority of rule when doing calculating Priority int `json:"priority"`