diff --git a/make/migrations/postgresql/0010_1.9.0_schema.up.sql b/make/migrations/postgresql/0010_1.9.0_schema.up.sql index 11c2248a6..47a71ed36 100644 --- a/make/migrations/postgresql/0010_1.9.0_schema.up.sql +++ b/make/migrations/postgresql/0010_1.9.0_schema.up.sql @@ -7,4 +7,52 @@ CREATE TABLE cve_whitelist ( expires_at bigint, items text NOT NULL, UNIQUE (project_id) -); \ No newline at end of file +); + +create table retention_policy +( + id serial PRIMARY KEY NOT NULL, + scope_level varchar(20), + scope_reference integer, + trigger_kind varchar(20), + data text, + create_time time, + update_time time +); + +create table retention_execution +( + id integer PRIMARY KEY NOT NULL, + policy_id integer, + status varchar(20), + status_text text, + dry boolean, + trigger varchar(20), + total integer, + succeed integer, + failed integer, + in_progress integer, + stopped integer, + start_time time, + end_time time +); + +create table retention_task +( + id integer PRIMARY KEY NOT NULL, + execution_id integer, + rule_id integer, + rule_display_text varchar(255), + artifact varchar(255), + timestamp time +); + +create table retention_schedule_job +( + id integer PRIMARY KEY NOT NULL, + status varchar(20), + policy_id integer, + job_id integer, + create_time time, + update_time time +); diff --git a/src/common/api/base.go b/src/common/api/base.go index fba8c3621..50cb6672d 100644 --- a/src/common/api/base.go +++ b/src/common/api/base.go @@ -131,6 +131,21 @@ func (b *BaseAPI) GetIDFromURL() (int64, error) { return id, nil } +// GetSpecialIDFromURL checks the ID with special name in request URL +func (b *BaseAPI) GetSpecialIDFromURL(name string) (int64, error) { + idStr := b.Ctx.Input.Param(":" + name) + if len(idStr) == 0 { + return 0, fmt.Errorf("invalid %s in URL", name) + } + + id, err := strconv.ParseInt(idStr, 10, 64) + if err != nil || id <= 0 { + return 0, errors.New("invalid ID in URL") + } + + return id, nil +} + // SetPaginationHeader set"Link" and "X-Total-Count" header for pagination request func (b *BaseAPI) SetPaginationHeader(total, page, pageSize int64) { b.Ctx.ResponseWriter.Header().Set("X-Total-Count", strconv.FormatInt(total, 10)) diff --git a/src/core/router.go b/src/core/router.go index 8170129d5..4249664b5 100644 --- a/src/core/router.go +++ b/src/core/router.go @@ -144,8 +144,8 @@ func initRouters() { beego.Router("/api/retentions", &retentionCtl.RetentionAPI{}, "post:CreateRetention") beego.Router("/api/retentions/:id", &retentionCtl.RetentionAPI{}, "put:UpdateRetention") beego.Router("/api/retentions/:id", &retentionCtl.RetentionAPI{}, "delete:DeleteRetention") - beego.Router("/api/retentions/:id/executions", &retentionCtl.RetentionAPI{}, "delete:TriggerRetentionExec") - beego.Router("/api/retentions/:id/executions/:eid", &retentionCtl.RetentionAPI{}, "delete:OperateRetentionExec") + beego.Router("/api/retentions/:id/executions", &retentionCtl.RetentionAPI{}, "post:TriggerRetentionExec") + beego.Router("/api/retentions/:id/executions/:eid", &retentionCtl.RetentionAPI{}, "put:OperateRetentionExec") beego.Router("/api/retentions/:id/executions/:eid", &retentionCtl.RetentionAPI{}, "get:GetRetentionExec") beego.Router("/api/retentions/:id/executions", &retentionCtl.RetentionAPI{}, "get:ListRetentionExec") beego.Router("/api/retentions/:id/executions/:eid/histories", &retentionCtl.RetentionAPI{}, "get:ListRetentionExecHistory") diff --git a/src/pkg/retention/controllers/retention.go b/src/pkg/retention/controllers/retention.go index 7f4eadc0a..da415ec80 100644 --- a/src/pkg/retention/controllers/retention.go +++ b/src/pkg/retention/controllers/retention.go @@ -3,19 +3,24 @@ package controllers import ( "github.com/goharbor/harbor/src/core/api" "github.com/goharbor/harbor/src/pkg/retention" + "github.com/goharbor/harbor/src/pkg/retention/policy" + "github.com/goharbor/harbor/src/pkg/retention/q" + "time" ) +// RetentionAPI ... type RetentionAPI struct { api.BaseController manager retention.Manager } // Prepare validates the user -func (t *RetentionAPI) Prepare() { - t.BaseController.Prepare() - t.manager = retention.NewManager() +func (r *RetentionAPI) Prepare() { + r.BaseController.Prepare() + r.manager = retention.NewManager() } +// GetRetention Get Retention func (r *RetentionAPI) GetRetention() { id, err := r.GetIDFromURL() if err != nil { @@ -31,19 +36,36 @@ func (r *RetentionAPI) GetRetention() { r.ServeJSON() } +// CreateRetention Create Retention func (r *RetentionAPI) CreateRetention() { - r.manager.CreatePolicy(nil) + p := &policy.Metadata{} + isValid, err := r.DecodeJSONReqAndValidate(p) + if !isValid { + r.SendBadRequestError(err) + return + } + + r.manager.CreatePolicy(p) } +// UpdateRetention Update Retention func (r *RetentionAPI) UpdateRetention() { - _, err := r.GetIDFromURL() + id, err := r.GetIDFromURL() if err != nil { r.SendBadRequestError(err) return } - r.manager.UpdatePolicy(nil) + p := &policy.Metadata{} + isValid, err := r.DecodeJSONReqAndValidate(p) + if !isValid { + r.SendBadRequestError(err) + return + } + p.ID = id + r.manager.UpdatePolicy(p) } +// DeleteRetention Delete Retention func (r *RetentionAPI) DeleteRetention() { id, err := r.GetIDFromURL() if err != nil { @@ -53,22 +75,95 @@ func (r *RetentionAPI) DeleteRetention() { r.manager.DeletePolicy(id) } +// TriggerRetentionExec Trigger Retention Execution func (r *RetentionAPI) TriggerRetentionExec() { - //r.manager.CreateExecution(nil) + id, err := r.GetIDFromURL() + if err != nil { + r.SendBadRequestError(err) + return + } + exec := &retention.Execution{ + PolicyID: id, + StartTime: time.Now(), + Status: "Running", + } + r.manager.CreateExecution(exec) } +// OperateRetentionExec Operate Retention Execution func (r *RetentionAPI) OperateRetentionExec() { + eid, err := r.GetSpecialIDFromURL("eid") + if err != nil { + r.SendBadRequestError(err) + return + } + exec := &retention.Execution{} + isValid, err := r.DecodeJSONReqAndValidate(exec) + if !isValid { + r.SendBadRequestError(err) + return + } + exec.ID = eid r.manager.UpdateExecution(nil) } +// GetRetentionExec Get Retention Execution func (r *RetentionAPI) GetRetentionExec() { - //r.manager.GetExecution(eid) + eid, err := r.GetSpecialIDFromURL("eid") + if err != nil { + r.SendBadRequestError(err) + return + } + exec, err := r.manager.GetExecution(eid) + if err != nil { + r.SendBadRequestError(err) + return + } + r.Data["json"] = exec + r.ServeJSON() } +// ListRetentionExec List Retention Execution func (r *RetentionAPI) ListRetentionExec() { - r.manager.ListExecutions(nil) + page, size, err := r.GetPaginationParams() + if err != nil { + r.SendInternalServerError(err) + return + } + query := &q.Query{ + PageNumber: page, + PageSize: size, + } + execs, err := r.manager.ListExecutions(query) + if err != nil { + r.SendBadRequestError(err) + return + } + r.Data["json"] = execs + r.ServeJSON() } +// ListRetentionExecHistory List Retention Execution Histories func (r *RetentionAPI) ListRetentionExecHistory() { - //r.manager.ListHistories(eid, nil) + eid, err := r.GetSpecialIDFromURL("eid") + if err != nil { + r.SendBadRequestError(err) + return + } + page, size, err := r.GetPaginationParams() + if err != nil { + r.SendInternalServerError(err) + return + } + query := &q.Query{ + PageNumber: page, + PageSize: size, + } + his, err := r.manager.ListHistories(eid, query) + if err != nil { + r.SendBadRequestError(err) + return + } + r.Data["json"] = his + r.ServeJSON() } diff --git a/src/pkg/retention/dao/models/retention.go b/src/pkg/retention/dao/models/retention.go index c8ad47b5e..ded0df53d 100644 --- a/src/pkg/retention/dao/models/retention.go +++ b/src/pkg/retention/dao/models/retention.go @@ -7,25 +7,27 @@ import ( func init() { orm.RegisterModel( - new (RetentionPolicy), - new (RetentionExecution), - new (RetentionTask), - new (RetentionScheduleJob), - ) + new(RetentionPolicy), + new(RetentionExecution), + new(RetentionTask), + new(RetentionScheduleJob), + ) } +// RetentionPolicy Retention Policy type RetentionPolicy struct { ID int64 `orm:"pk;auto;column(id)" json:"id"` // 'system', 'project' and 'repository' - ScopeLevel string - ScopeReference int64 - TriggerKind string + ScopeLevel string + ScopeReference int64 + TriggerKind string // json format, include algorithm, rules, exclusions Data string CreateTime time.Time UpdateTime time.Time } +// RetentionExecution Retention Execution type RetentionExecution struct { ID int64 `orm:"pk;auto;column(id)" json:"id"` PolicyID int64 @@ -43,17 +45,17 @@ type RetentionExecution struct { EndTime time.Time } +// RetentionTask Retention Task type RetentionTask struct { - ID int64 - ExecutionID int64 - // image, chart - ResourceType string - Resource string - Status string - StartTime time.Time - EndTime time.Time + ID int64 + ExecutionID int64 + RuleID int + RuleDisplayText string + Artifact string + Timestamp time.Time } +// RetentionScheduleJob Retention Schedule Job type RetentionScheduleJob struct { ID int64 Status string diff --git a/src/pkg/retention/dao/retention.go b/src/pkg/retention/dao/retention.go index 8630da3cd..354e387e2 100644 --- a/src/pkg/retention/dao/retention.go +++ b/src/pkg/retention/dao/retention.go @@ -3,19 +3,23 @@ package dao import ( "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/pkg/retention/dao/models" + "github.com/goharbor/harbor/src/pkg/retention/q" ) +// CreatePolicy Create Policy func CreatePolicy(p *models.RetentionPolicy) (int64, error) { o := dao.GetOrmer() return o.Insert(p) } +// UpdatePolicy Update Policy func UpdatePolicy(p *models.RetentionPolicy) error { o := dao.GetOrmer() _, err := o.Update(p) return err } +// DeletePolicy Delete Policy func DeletePolicy(id int64) error { o := dao.GetOrmer() _, err := o.Delete(&models.RetentionPolicy{ @@ -24,6 +28,7 @@ func DeletePolicy(id int64) error { return err } +// GetPolicy Get Policy func GetPolicy(id int64) (*models.RetentionPolicy, error) { o := dao.GetOrmer() p := &models.RetentionPolicy{ @@ -31,7 +36,73 @@ func GetPolicy(id int64) (*models.RetentionPolicy, error) { } if err := o.Read(p); err != nil { return nil, err - } else { - return p, nil } + return p, nil +} + +// CreateExecution Create Execution +func CreateExecution(e *models.RetentionExecution) (int64, error) { + o := dao.GetOrmer() + return o.Insert(e) +} + +// UpdateExecution Update Execution +func UpdateExecution(e *models.RetentionExecution) error { + o := dao.GetOrmer() + _, err := o.Update(e) + return err +} + +// DeleteExecution Delete Execution +func DeleteExecution(id int64) error { + o := dao.GetOrmer() + _, err := o.Delete(&models.RetentionExecution{ + ID: id, + }) + return err +} + +// GetExecution Get Execution +func GetExecution(id int64) (*models.RetentionExecution, error) { + o := dao.GetOrmer() + e := &models.RetentionExecution{ + ID: id, + } + if err := o.Read(e); err != nil { + return nil, err + } + return e, nil +} + +// ListExecutions List Executions +func ListExecutions(query *q.Query) ([]*models.RetentionExecution, error) { + o := dao.GetOrmer() + qs := o.QueryTable(new(models.RetentionExecution)) + qs.Limit(query.PageSize, (query.PageNumber-1)*query.PageSize) + var execs []*models.RetentionExecution + _, err := qs.All(&execs) + if err != nil { + return nil, err + } + return execs, nil +} + +// ListExecHistories List Execution Histories +func ListExecHistories(executionID int64, query *q.Query) ([]*models.RetentionTask, error) { + o := dao.GetOrmer() + qs := o.QueryTable(new(models.RetentionTask)) + qs.Filter("Execution_ID", executionID) + qs.Limit(query.PageSize, (query.PageNumber-1)*query.PageSize) + var tasks []*models.RetentionTask + _, err := qs.All(&tasks) + if err != nil { + return nil, err + } + return tasks, nil +} + +// AppendExecHistory Append Execution History +func AppendExecHistory(t *models.RetentionTask) (int64, error) { + o := dao.GetOrmer() + return o.Insert(t) } diff --git a/src/pkg/retention/dao/retention_test.go b/src/pkg/retention/dao/retention_test.go index bda3bd602..059711cb5 100644 --- a/src/pkg/retention/dao/retention_test.go +++ b/src/pkg/retention/dao/retention_test.go @@ -21,7 +21,6 @@ func TestMain(m *testing.M) { func TestPolicy(t *testing.T) { p := &policy.Metadata{ - ID: 1, Algorithm: "OR", Rules: []rule.Metadata{ { @@ -43,11 +42,13 @@ func TestPolicy(t *testing.T) { Pattern: "release-[\\d\\.]+", }, }, - ScopeSelectors: []*rule.Selector{ - { - Kind: "regularExpression", - Decoration: "matches", - Pattern: ".+", + ScopeSelectors: map[string][]*rule.Selector{ + "repository": { + { + Kind: "regularExpression", + Decoration: "matches", + Pattern: ".+", + }, }, }, }, @@ -79,6 +80,7 @@ func TestPolicy(t *testing.T) { p1, err = GetPolicy(id) assert.Nil(t, err) assert.EqualValues(t, "project", p1.ScopeLevel) + assert.True(t, p1.ID > 0) p1.ScopeLevel = "test" err = UpdatePolicy(p1) diff --git a/src/pkg/retention/manager.go b/src/pkg/retention/manager.go index 9470db2b8..ad37ff3f8 100644 --- a/src/pkg/retention/manager.go +++ b/src/pkg/retention/manager.go @@ -36,7 +36,7 @@ type Manager interface { // Get the specified policy GetPolicy(ID int64) (*policy.Metadata, error) // Create a new retention execution - CreateExecution(execution *Execution) (string, error) + CreateExecution(execution *Execution) (int64, error) // Update the specified execution UpdateExecution(execution *Execution) error // Get the specified execution @@ -49,9 +49,11 @@ type Manager interface { ListHistories(executionID int64, query *q.Query) ([]*History, error) } +// DefaultManager ... type DefaultManager struct { } +// CreatePolicy Create Policy func (d *DefaultManager) CreatePolicy(p *policy.Metadata) (int64, error) { var p1 *models.RetentionPolicy p1.ScopeLevel = p.Scope.Level @@ -63,86 +65,123 @@ func (d *DefaultManager) CreatePolicy(p *policy.Metadata) (int64, error) { return dao.CreatePolicy(p1) } +// UpdatePolicy Update Policy func (d *DefaultManager) UpdatePolicy(p *policy.Metadata) error { var p1 *models.RetentionPolicy p1.ID = p.ID p1.ScopeLevel = p.Scope.Level p1.TriggerKind = p.Trigger.Kind + p.ID = 0 data, _ := json.Marshal(p) + p.ID = p1.ID p1.Data = string(data) p1.UpdateTime = time.Now() return dao.UpdatePolicy(p1) } +// DeletePolicy Delete Policy func (d *DefaultManager) DeletePolicy(id int64) error { return dao.DeletePolicy(id) } +// GetPolicy Get Policy func (d *DefaultManager) GetPolicy(id int64) (*policy.Metadata, error) { - if p1,err:=dao.GetPolicy(id);err!=nil{ - return nil,err - }else{ - var p *policy.Metadata - if err=json.Unmarshal([]byte(p1.Data), p);err!=nil{ - return nil,err - }else{ - return p,nil - } + p1, err := dao.GetPolicy(id) + if err != nil { + return nil, err } + var p *policy.Metadata + if err = json.Unmarshal([]byte(p1.Data), p); err != nil { + return nil, err + } + return p, nil } -func (d *DefaultManager) CreateExecution(execution *Execution) (string, error) { - panic("implement me") +// CreateExecution Create Execution +func (d *DefaultManager) CreateExecution(execution *Execution) (int64, error) { + var exec *models.RetentionExecution + exec.PolicyID = execution.PolicyID + exec.StartTime = time.Now() + exec.Status = "Running" + return dao.CreateExecution(exec) } +// UpdateExecution Update Execution func (d *DefaultManager) UpdateExecution(execution *Execution) error { - panic("implement me") + var exec *models.RetentionExecution + exec.ID = execution.ID + exec.PolicyID = execution.PolicyID + exec.StartTime = time.Now() + exec.Status = "Running" + return dao.UpdateExecution(exec) } +// ListExecutions List Executions func (d *DefaultManager) ListExecutions(query *q.Query) ([]*Execution, error) { - return []*Execution{ - { - ID: 1, - PolicyID: 1, - StartTime: time.Now().Add(-time.Minute), - EndTime: time.Now(), - Status: "Success", - }, - { - ID: 2, - PolicyID: 1, - StartTime: time.Now().Add(-time.Minute), - EndTime: time.Now(), - Status: "Failed", - }, - { - ID: 3, - PolicyID: 1, - StartTime: time.Now().Add(-time.Minute), - EndTime: time.Now(), - Status: "Running", - }, - }, nil + execs, err := dao.ListExecutions(query) + if err != nil { + return nil, err + } + var execs1 []*Execution + for _, e := range execs { + var e1 *Execution + e1.ID = e.ID + e1.PolicyID = e.PolicyID + e1.Status = e.Status + e1.StartTime = e.StartTime + e1.EndTime = e.EndTime + execs1 = append(execs1, e1) + } + return execs1, nil } +// GetExecution Get Execution func (d *DefaultManager) GetExecution(eid int64) (*Execution, error) { - return &Execution{ - ID: 1, - PolicyID: 1, - StartTime: time.Now().Add(-time.Minute), - EndTime: time.Now(), - Status: "Success", - }, nil + e, err := dao.GetExecution(eid) + if err != nil { + return nil, err + } + var e1 *Execution + e1.ID = e.ID + e1.PolicyID = e.PolicyID + e1.Status = e.Status + e1.StartTime = e.StartTime + e1.EndTime = e.EndTime + return e1, nil } +// ListHistories List Histories func (d *DefaultManager) ListHistories(executionID int64, query *q.Query) ([]*History, error) { - panic("implement me") + his, err := dao.ListExecHistories(executionID, query) + if err != nil { + return nil, err + } + var his1 []*History + for _, h := range his { + var h1 *History + h1.ExecutionID = h.ExecutionID + h1.Artifact = h.Artifact + h1.Rule.ID = h.RuleID + h1.Rule.DisplayText = h.RuleDisplayText + h1.Timestamp = h.Timestamp + his1 = append(his1, h1) + } + return his1, nil } -func (d *DefaultManager) AppendHistory(history *History) error { - panic("implement me") +// AppendHistory Append History +func (d *DefaultManager) AppendHistory(h *History) error { + var h1 *models.RetentionTask + h1.ExecutionID = h.ExecutionID + h1.Artifact = h.Artifact + h1.RuleID = h.Rule.ID + h1.RuleDisplayText = h.Rule.DisplayText + h1.Timestamp = h.Timestamp + _, err := dao.AppendExecHistory(h1) + return err } +// NewManager ... func NewManager() Manager { return &DefaultManager{} } diff --git a/src/pkg/retention/models.go b/src/pkg/retention/models.go index ba907fe83..82c7351b3 100644 --- a/src/pkg/retention/models.go +++ b/src/pkg/retention/models.go @@ -18,10 +18,10 @@ import "time" // Execution of retention type Execution struct { - ID int `json:"id"` - PolicyID int `json:"policy_id"` + ID int64 `json:"id,omitempty"` + PolicyID int64 `json:"policy_id"` StartTime time.Time `json:"start_time"` - EndTime time.Time `json:"end_time"` + EndTime time.Time `json:"end_time,omitempty"` Status string `json:"status"` } @@ -35,7 +35,7 @@ type TaskSubmitResult struct { // History of retention type History struct { - ExecutionID string `json:"execution_id"` + ExecutionID int64 `json:"execution_id"` Rule struct { ID int `json:"id"` DisplayText string `json:"display_text"` diff --git a/src/pkg/retention/policy/models.go b/src/pkg/retention/policy/models.go index e25804240..38541a3e0 100644 --- a/src/pkg/retention/policy/models.go +++ b/src/pkg/retention/policy/models.go @@ -26,7 +26,7 @@ const ( // Metadata of policy type Metadata struct { // UUID of the policy - ID int64 `json:"-"` + ID int64 `json:"id,omitempty"` // Algorithm applied to the rules // "OR" / "AND" diff --git a/src/pkg/retention/q/query.go b/src/pkg/retention/q/query.go index a7c875666..db337fe13 100644 --- a/src/pkg/retention/q/query.go +++ b/src/pkg/retention/q/query.go @@ -16,6 +16,6 @@ package q // Query parameters type Query struct { - PageNumber int - PageSize int + PageNumber int64 + PageSize int64 }