Merge pull request #8296 from bitsf/tag_retention_controller

Tag retention controller

make/migrations/postgresql/0010_1.9.0_schema.up.sql

@@ -8,3 +8,51 @@ CREATE TABLE cve_whitelist (
     items text NOT NULL,
     UNIQUE (project_id)
 );
+
+create table retention_policy
+(
+	id serial PRIMARY KEY NOT NULL,
+	scope_level varchar(20),
+	scope_reference integer,
+	trigger_kind varchar(20),
+	data text,
+	create_time time,
+	update_time time
+);
+
+create table retention_execution
+(
+	id integer PRIMARY KEY NOT NULL,
+	policy_id integer,
+	status varchar(20),
+	status_text text,
+	dry boolean,
+	trigger varchar(20),
+	total integer,
+	succeed integer,
+	failed integer,
+	in_progress integer,
+	stopped integer,
+	start_time time,
+	end_time time
+);
+
+create table retention_task
+(
+	id integer PRIMARY KEY NOT NULL,
+	execution_id integer,
+	rule_id integer,
+	rule_display_text varchar(255),
+	artifact varchar(255),
+	timestamp time
+);
+
+create table retention_schedule_job
+(
+	id integer PRIMARY KEY NOT NULL,
+	status varchar(20),
+	policy_id integer,
+	job_id integer,
+	create_time time,
+	update_time time
+);

src/common/api/base.go

@@ -131,6 +131,21 @@ func (b *BaseAPI) GetIDFromURL() (int64, error) {
 	return id, nil
 }
 
+// GetSpecialIDFromURL checks the ID with special name in request URL
+func (b *BaseAPI) GetSpecialIDFromURL(name string) (int64, error) {
+	idStr := b.Ctx.Input.Param(":" + name)
+	if len(idStr) == 0 {
+		return 0, fmt.Errorf("invalid %s in URL", name)
+	}
+
+	id, err := strconv.ParseInt(idStr, 10, 64)
+	if err != nil || id <= 0 {
+		return 0, errors.New("invalid ID in URL")
+	}
+
+	return id, nil
+}
+
 // SetPaginationHeader set"Link" and "X-Total-Count" header for pagination request
 func (b *BaseAPI) SetPaginationHeader(total, page, pageSize int64) {
 	b.Ctx.ResponseWriter.Header().Set("X-Total-Count", strconv.FormatInt(total, 10))

src/core/router.go

@@ -144,8 +144,8 @@ func initRouters() {
 	beego.Router("/api/retentions", &retentionCtl.RetentionAPI{}, "post:CreateRetention")
 	beego.Router("/api/retentions/:id", &retentionCtl.RetentionAPI{}, "put:UpdateRetention")
 	beego.Router("/api/retentions/:id", &retentionCtl.RetentionAPI{}, "delete:DeleteRetention")
-	beego.Router("/api/retentions/:id/executions", &retentionCtl.RetentionAPI{}, "delete:TriggerRetentionExec")
+	beego.Router("/api/retentions/:id/executions", &retentionCtl.RetentionAPI{}, "post:TriggerRetentionExec")
-	beego.Router("/api/retentions/:id/executions/:eid", &retentionCtl.RetentionAPI{}, "delete:OperateRetentionExec")
+	beego.Router("/api/retentions/:id/executions/:eid", &retentionCtl.RetentionAPI{}, "put:OperateRetentionExec")
 	beego.Router("/api/retentions/:id/executions/:eid", &retentionCtl.RetentionAPI{}, "get:GetRetentionExec")
 	beego.Router("/api/retentions/:id/executions", &retentionCtl.RetentionAPI{}, "get:ListRetentionExec")
 	beego.Router("/api/retentions/:id/executions/:eid/histories", &retentionCtl.RetentionAPI{}, "get:ListRetentionExecHistory")

src/pkg/retention/controllers/retention.go

@@ -3,19 +3,24 @@ package controllers
 import (
 	"github.com/goharbor/harbor/src/core/api"
 	"github.com/goharbor/harbor/src/pkg/retention"
+	"github.com/goharbor/harbor/src/pkg/retention/policy"
+	"github.com/goharbor/harbor/src/pkg/retention/q"
+	"time"
 )
 
+// RetentionAPI ...
 type RetentionAPI struct {
 	api.BaseController
 	manager retention.Manager
 }
 
 // Prepare validates the user
-func (t *RetentionAPI) Prepare() {
+func (r *RetentionAPI) Prepare() {
-	t.BaseController.Prepare()
+	r.BaseController.Prepare()
-	t.manager = retention.NewManager()
+	r.manager = retention.NewManager()
 }
 
+// GetRetention Get Retention
 func (r *RetentionAPI) GetRetention() {
 	id, err := r.GetIDFromURL()
 	if err != nil {
@@ -31,19 +36,36 @@ func (r *RetentionAPI) GetRetention() {
 	r.ServeJSON()
 }
 
+// CreateRetention Create Retention
 func (r *RetentionAPI) CreateRetention() {
-	r.manager.CreatePolicy(nil)
+	p := &policy.Metadata{}
+	isValid, err := r.DecodeJSONReqAndValidate(p)
+	if !isValid {
+		r.SendBadRequestError(err)
+		return
+	}
+
+	r.manager.CreatePolicy(p)
 }
 
+// UpdateRetention Update Retention
 func (r *RetentionAPI) UpdateRetention() {
-	_, err := r.GetIDFromURL()
+	id, err := r.GetIDFromURL()
 	if err != nil {
 		r.SendBadRequestError(err)
 		return
 	}
-	r.manager.UpdatePolicy(nil)
+	p := &policy.Metadata{}
+	isValid, err := r.DecodeJSONReqAndValidate(p)
+	if !isValid {
+		r.SendBadRequestError(err)
+		return
+	}
+	p.ID = id
+	r.manager.UpdatePolicy(p)
 }
 
+// DeleteRetention Delete Retention
 func (r *RetentionAPI) DeleteRetention() {
 	id, err := r.GetIDFromURL()
 	if err != nil {
@@ -53,22 +75,95 @@ func (r *RetentionAPI) DeleteRetention() {
 	r.manager.DeletePolicy(id)
 }
 
+// TriggerRetentionExec Trigger Retention Execution
 func (r *RetentionAPI) TriggerRetentionExec() {
-	//r.manager.CreateExecution(nil)
+	id, err := r.GetIDFromURL()
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	exec := &retention.Execution{
+		PolicyID:  id,
+		StartTime: time.Now(),
+		Status:    "Running",
+	}
+	r.manager.CreateExecution(exec)
 }
 
+// OperateRetentionExec Operate Retention Execution
 func (r *RetentionAPI) OperateRetentionExec() {
+	eid, err := r.GetSpecialIDFromURL("eid")
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	exec := &retention.Execution{}
+	isValid, err := r.DecodeJSONReqAndValidate(exec)
+	if !isValid {
+		r.SendBadRequestError(err)
+		return
+	}
+	exec.ID = eid
 	r.manager.UpdateExecution(nil)
 }
 
+// GetRetentionExec Get Retention Execution
 func (r *RetentionAPI) GetRetentionExec() {
-	//r.manager.GetExecution(eid)
+	eid, err := r.GetSpecialIDFromURL("eid")
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	exec, err := r.manager.GetExecution(eid)
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	r.Data["json"] = exec
+	r.ServeJSON()
 }
 
+// ListRetentionExec List Retention Execution
 func (r *RetentionAPI) ListRetentionExec() {
-	r.manager.ListExecutions(nil)
+	page, size, err := r.GetPaginationParams()
+	if err != nil {
+		r.SendInternalServerError(err)
+		return
+	}
+	query := &q.Query{
+		PageNumber: page,
+		PageSize:   size,
+	}
+	execs, err := r.manager.ListExecutions(query)
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	r.Data["json"] = execs
+	r.ServeJSON()
 }
 
+// ListRetentionExecHistory List Retention Execution Histories
 func (r *RetentionAPI) ListRetentionExecHistory() {
-	//r.manager.ListHistories(eid, nil)
+	eid, err := r.GetSpecialIDFromURL("eid")
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	page, size, err := r.GetPaginationParams()
+	if err != nil {
+		r.SendInternalServerError(err)
+		return
+	}
+	query := &q.Query{
+		PageNumber: page,
+		PageSize:   size,
+	}
+	his, err := r.manager.ListHistories(eid, query)
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	r.Data["json"] = his
+	r.ServeJSON()
 }

src/pkg/retention/dao/models/retention.go

@@ -7,13 +7,14 @@ import (
 
 func init() {
 	orm.RegisterModel(
-		new (RetentionPolicy),
+		new(RetentionPolicy),
-		new (RetentionExecution),
+		new(RetentionExecution),
-		new (RetentionTask),
+		new(RetentionTask),
-		new (RetentionScheduleJob),
+		new(RetentionScheduleJob),
 	)
 }
 
+// RetentionPolicy Retention Policy
 type RetentionPolicy struct {
 	ID int64 `orm:"pk;auto;column(id)" json:"id"`
 	// 'system', 'project' and 'repository'
@@ -26,6 +27,7 @@ type RetentionPolicy struct {
 	UpdateTime time.Time
 }
 
+// RetentionExecution Retention Execution
 type RetentionExecution struct {
 	ID         int64 `orm:"pk;auto;column(id)" json:"id"`
 	PolicyID   int64
@@ -43,17 +45,17 @@ type RetentionExecution struct {
 	EndTime    time.Time
 }
 
+// RetentionTask Retention Task
 type RetentionTask struct {
 	ID              int64
 	ExecutionID     int64
-	// image, chart
+	RuleID          int
-	ResourceType string
+	RuleDisplayText string
-	Resource     string
+	Artifact        string
-	Status       string
+	Timestamp       time.Time
-	StartTime    time.Time
-	EndTime      time.Time
 }
 
+// RetentionScheduleJob Retention Schedule Job
 type RetentionScheduleJob struct {
 	ID         int64
 	Status     string

src/pkg/retention/dao/retention.go

@@ -3,19 +3,23 @@ package dao
 import (
 	"github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/pkg/retention/dao/models"
+	"github.com/goharbor/harbor/src/pkg/retention/q"
 )
 
+// CreatePolicy Create Policy
 func CreatePolicy(p *models.RetentionPolicy) (int64, error) {
 	o := dao.GetOrmer()
 	return o.Insert(p)
 }
 
+// UpdatePolicy Update Policy
 func UpdatePolicy(p *models.RetentionPolicy) error {
 	o := dao.GetOrmer()
 	_, err := o.Update(p)
 	return err
 }
 
+// DeletePolicy Delete Policy
 func DeletePolicy(id int64) error {
 	o := dao.GetOrmer()
 	_, err := o.Delete(&models.RetentionPolicy{
@@ -24,6 +28,7 @@ func DeletePolicy(id int64) error {
 	return err
 }
 
+// GetPolicy Get Policy
 func GetPolicy(id int64) (*models.RetentionPolicy, error) {
 	o := dao.GetOrmer()
 	p := &models.RetentionPolicy{
@@ -31,7 +36,73 @@ func GetPolicy(id int64) (*models.RetentionPolicy, error) {
 	}
 	if err := o.Read(p); err != nil {
 		return nil, err
-	} else {
-		return p, nil
 	}
+	return p, nil
+}
+
+// CreateExecution Create Execution
+func CreateExecution(e *models.RetentionExecution) (int64, error) {
+	o := dao.GetOrmer()
+	return o.Insert(e)
+}
+
+// UpdateExecution Update Execution
+func UpdateExecution(e *models.RetentionExecution) error {
+	o := dao.GetOrmer()
+	_, err := o.Update(e)
+	return err
+}
+
+// DeleteExecution Delete Execution
+func DeleteExecution(id int64) error {
+	o := dao.GetOrmer()
+	_, err := o.Delete(&models.RetentionExecution{
+		ID: id,
+	})
+	return err
+}
+
+// GetExecution Get Execution
+func GetExecution(id int64) (*models.RetentionExecution, error) {
+	o := dao.GetOrmer()
+	e := &models.RetentionExecution{
+		ID: id,
+	}
+	if err := o.Read(e); err != nil {
+		return nil, err
+	}
+	return e, nil
+}
+
+// ListExecutions List Executions
+func ListExecutions(query *q.Query) ([]*models.RetentionExecution, error) {
+	o := dao.GetOrmer()
+	qs := o.QueryTable(new(models.RetentionExecution))
+	qs.Limit(query.PageSize, (query.PageNumber-1)*query.PageSize)
+	var execs []*models.RetentionExecution
+	_, err := qs.All(&execs)
+	if err != nil {
+		return nil, err
+	}
+	return execs, nil
+}
+
+// ListExecHistories List Execution Histories
+func ListExecHistories(executionID int64, query *q.Query) ([]*models.RetentionTask, error) {
+	o := dao.GetOrmer()
+	qs := o.QueryTable(new(models.RetentionTask))
+	qs.Filter("Execution_ID", executionID)
+	qs.Limit(query.PageSize, (query.PageNumber-1)*query.PageSize)
+	var tasks []*models.RetentionTask
+	_, err := qs.All(&tasks)
+	if err != nil {
+		return nil, err
+	}
+	return tasks, nil
+}
+
+// AppendExecHistory Append Execution History
+func AppendExecHistory(t *models.RetentionTask) (int64, error) {
+	o := dao.GetOrmer()
+	return o.Insert(t)
 }

src/pkg/retention/dao/retention_test.go

@@ -21,7 +21,6 @@ func TestMain(m *testing.M) {
 
 func TestPolicy(t *testing.T) {
 	p := &policy.Metadata{
-		ID:        1,
 		Algorithm: "OR",
 		Rules: []rule.Metadata{
 			{
@@ -43,7 +42,8 @@ func TestPolicy(t *testing.T) {
 						Pattern:    "release-[\\d\\.]+",
 					},
 				},
-				ScopeSelectors: []*rule.Selector{
+				ScopeSelectors: map[string][]*rule.Selector{
+					"repository": {
 						{
 							Kind:       "regularExpression",
 							Decoration: "matches",
@@ -52,6 +52,7 @@ func TestPolicy(t *testing.T) {
 					},
 				},
 			},
+		},
 		Trigger: &policy.Trigger{
 			Kind: "Schedule",
 			Settings: map[string]interface{}{
@@ -79,6 +80,7 @@ func TestPolicy(t *testing.T) {
 	p1, err = GetPolicy(id)
 	assert.Nil(t, err)
 	assert.EqualValues(t, "project", p1.ScopeLevel)
+	assert.True(t, p1.ID > 0)
 
 	p1.ScopeLevel = "test"
 	err = UpdatePolicy(p1)

src/pkg/retention/manager.go

@@ -36,7 +36,7 @@ type Manager interface {
 	// Get the specified policy
 	GetPolicy(ID int64) (*policy.Metadata, error)
 	// Create a new retention execution
-	CreateExecution(execution *Execution) (string, error)
+	CreateExecution(execution *Execution) (int64, error)
 	// Update the specified execution
 	UpdateExecution(execution *Execution) error
 	// Get the specified execution
@@ -49,9 +49,11 @@ type Manager interface {
 	ListHistories(executionID int64, query *q.Query) ([]*History, error)
 }
 
+// DefaultManager ...
 type DefaultManager struct {
 }
 
+// CreatePolicy Create Policy
 func (d *DefaultManager) CreatePolicy(p *policy.Metadata) (int64, error) {
 	var p1 *models.RetentionPolicy
 	p1.ScopeLevel = p.Scope.Level
@@ -63,86 +65,123 @@ func (d *DefaultManager) CreatePolicy(p *policy.Metadata) (int64, error) {
 	return dao.CreatePolicy(p1)
 }
 
+// UpdatePolicy Update Policy
 func (d *DefaultManager) UpdatePolicy(p *policy.Metadata) error {
 	var p1 *models.RetentionPolicy
 	p1.ID = p.ID
 	p1.ScopeLevel = p.Scope.Level
 	p1.TriggerKind = p.Trigger.Kind
+	p.ID = 0
 	data, _ := json.Marshal(p)
+	p.ID = p1.ID
 	p1.Data = string(data)
 	p1.UpdateTime = time.Now()
 	return dao.UpdatePolicy(p1)
 }
 
+// DeletePolicy Delete Policy
 func (d *DefaultManager) DeletePolicy(id int64) error {
 	return dao.DeletePolicy(id)
 }
 
+// GetPolicy Get Policy
 func (d *DefaultManager) GetPolicy(id int64) (*policy.Metadata, error) {
-	if p1,err:=dao.GetPolicy(id);err!=nil{
+	p1, err := dao.GetPolicy(id)
-		return nil,err
+	if err != nil {
-	}else{
+		return nil, err
+	}
 	var p *policy.Metadata
-		if err=json.Unmarshal([]byte(p1.Data), p);err!=nil{
+	if err = json.Unmarshal([]byte(p1.Data), p); err != nil {
-			return nil,err
+		return nil, err
-		}else{
-			return p,nil
-		}
 	}
+	return p, nil
 }
 
-func (d *DefaultManager) CreateExecution(execution *Execution) (string, error) {
+// CreateExecution Create Execution
-	panic("implement me")
+func (d *DefaultManager) CreateExecution(execution *Execution) (int64, error) {
+	var exec *models.RetentionExecution
+	exec.PolicyID = execution.PolicyID
+	exec.StartTime = time.Now()
+	exec.Status = "Running"
+	return dao.CreateExecution(exec)
 }
 
+// UpdateExecution Update Execution
 func (d *DefaultManager) UpdateExecution(execution *Execution) error {
-	panic("implement me")
+	var exec *models.RetentionExecution
+	exec.ID = execution.ID
+	exec.PolicyID = execution.PolicyID
+	exec.StartTime = time.Now()
+	exec.Status = "Running"
+	return dao.UpdateExecution(exec)
 }
 
+// ListExecutions List Executions
 func (d *DefaultManager) ListExecutions(query *q.Query) ([]*Execution, error) {
-	return []*Execution{
+	execs, err := dao.ListExecutions(query)
-		{
+	if err != nil {
-			ID:        1,
+		return nil, err
-			PolicyID:  1,
+	}
-			StartTime: time.Now().Add(-time.Minute),
+	var execs1 []*Execution
-			EndTime:   time.Now(),
+	for _, e := range execs {
-			Status:    "Success",
+		var e1 *Execution
-		},
+		e1.ID = e.ID
-		{
+		e1.PolicyID = e.PolicyID
-			ID:        2,
+		e1.Status = e.Status
-			PolicyID:  1,
+		e1.StartTime = e.StartTime
-			StartTime: time.Now().Add(-time.Minute),
+		e1.EndTime = e.EndTime
-			EndTime:   time.Now(),
+		execs1 = append(execs1, e1)
-			Status:    "Failed",
+	}
-		},
+	return execs1, nil
-		{
-			ID:        3,
-			PolicyID:  1,
-			StartTime: time.Now().Add(-time.Minute),
-			EndTime:   time.Now(),
-			Status:    "Running",
-		},
-	}, nil
 }
 
+// GetExecution Get Execution
 func (d *DefaultManager) GetExecution(eid int64) (*Execution, error) {
-	return &Execution{
+	e, err := dao.GetExecution(eid)
-		ID:        1,
+	if err != nil {
-		PolicyID:  1,
+		return nil, err
-		StartTime: time.Now().Add(-time.Minute),
+	}
-		EndTime:   time.Now(),
+	var e1 *Execution
-		Status:    "Success",
+	e1.ID = e.ID
-	}, nil
+	e1.PolicyID = e.PolicyID
+	e1.Status = e.Status
+	e1.StartTime = e.StartTime
+	e1.EndTime = e.EndTime
+	return e1, nil
 }
 
+// ListHistories List Histories
 func (d *DefaultManager) ListHistories(executionID int64, query *q.Query) ([]*History, error) {
-	panic("implement me")
+	his, err := dao.ListExecHistories(executionID, query)
+	if err != nil {
+		return nil, err
+	}
+	var his1 []*History
+	for _, h := range his {
+		var h1 *History
+		h1.ExecutionID = h.ExecutionID
+		h1.Artifact = h.Artifact
+		h1.Rule.ID = h.RuleID
+		h1.Rule.DisplayText = h.RuleDisplayText
+		h1.Timestamp = h.Timestamp
+		his1 = append(his1, h1)
+	}
+	return his1, nil
 }
 
-func (d *DefaultManager) AppendHistory(history *History) error {
+// AppendHistory Append History
-	panic("implement me")
+func (d *DefaultManager) AppendHistory(h *History) error {
+	var h1 *models.RetentionTask
+	h1.ExecutionID = h.ExecutionID
+	h1.Artifact = h.Artifact
+	h1.RuleID = h.Rule.ID
+	h1.RuleDisplayText = h.Rule.DisplayText
+	h1.Timestamp = h.Timestamp
+	_, err := dao.AppendExecHistory(h1)
+	return err
 }
 
+// NewManager ...
 func NewManager() Manager {
 	return &DefaultManager{}
 }

src/pkg/retention/models.go

@@ -18,10 +18,10 @@ import "time"
 
 // Execution of retention
 type Execution struct {
-	ID        int       `json:"id"`
+	ID        int64     `json:"id,omitempty"`
-	PolicyID  int    `json:"policy_id"`
+	PolicyID  int64     `json:"policy_id"`
 	StartTime time.Time `json:"start_time"`
-	EndTime   time.Time `json:"end_time"`
+	EndTime   time.Time `json:"end_time,omitempty"`
 	Status    string    `json:"status"`
 }
 
@@ -35,7 +35,7 @@ type TaskSubmitResult struct {
 
 // History of retention
 type History struct {
-	ExecutionID string `json:"execution_id"`
+	ExecutionID int64 `json:"execution_id"`
 	Rule        struct {
 		ID          int    `json:"id"`
 		DisplayText string `json:"display_text"`

src/pkg/retention/policy/models.go

@@ -26,7 +26,7 @@ const (
 // Metadata of policy
 type Metadata struct {
 	// UUID of the policy
-	ID int64 `json:"-"`
+	ID int64 `json:"id,omitempty"`
 
 	// Algorithm applied to the rules
 	// "OR" / "AND"

src/pkg/retention/q/query.go

@@ -16,6 +16,6 @@ package q
 
 // Query parameters
 type Query struct {
-	PageNumber int
+	PageNumber int64
-	PageSize   int
+	PageSize   int64
 }