retention api

Change-Id: I70f2c34d6bb96ecf4cb5359e2b1ab2dbb99fdbf9
Signed-off-by: Ziming Zhang <zziming@vmware.com>

src/common/api/base.go

@@ -131,6 +131,21 @@ func (b *BaseAPI) GetIDFromURL() (int64, error) {
 	return id, nil
 }
 
+// GetIDFromURL checks the ID in request URL
+func (b *BaseAPI) GetSpecialIDFromURL(name string) (int64, error) {
+	idStr := b.Ctx.Input.Param(":"+name)
+	if len(idStr) == 0 {
+		return 0, errors.New(fmt.Sprintf("invalid %s in URL", name))
+	}
+
+	id, err := strconv.ParseInt(idStr, 10, 64)
+	if err != nil || id <= 0 {
+		return 0, errors.New("invalid ID in URL")
+	}
+
+	return id, nil
+}
+
 // SetPaginationHeader set"Link" and "X-Total-Count" header for pagination request
 func (b *BaseAPI) SetPaginationHeader(total, page, pageSize int64) {
 	b.Ctx.ResponseWriter.Header().Set("X-Total-Count", strconv.FormatInt(total, 10))

src/core/router.go

@@ -24,6 +24,7 @@ import (
 	"github.com/goharbor/harbor/src/core/service/notifications/jobs"
 	"github.com/goharbor/harbor/src/core/service/notifications/registry"
 	"github.com/goharbor/harbor/src/core/service/token"
+	retentionCtl "github.com/goharbor/harbor/src/pkg/retention/controllers"
 
 	"github.com/astaxie/beego"
 )
@@ -139,6 +140,16 @@ func initRouters() {
 	beego.Router("/api/registries/:id/info", &api.RegistryAPI{}, "get:GetInfo")
 	beego.Router("/api/registries/:id/namespace", &api.RegistryAPI{}, "get:GetNamespace")
 
+	beego.Router("/api/retentions/:id", &retentionCtl.RetentionAPI{}, "get:GetRetention")
+	beego.Router("/api/retentions", &retentionCtl.RetentionAPI{}, "post:CreateRetention")
+	beego.Router("/api/retentions/:id", &retentionCtl.RetentionAPI{}, "put:UpdateRetention")
+	beego.Router("/api/retentions/:id", &retentionCtl.RetentionAPI{}, "delete:DeleteRetention")
+	beego.Router("/api/retentions/:id/executions", &retentionCtl.RetentionAPI{}, "post:TriggerRetentionExec")
+	beego.Router("/api/retentions/:id/executions/:eid", &retentionCtl.RetentionAPI{}, "put:OperateRetentionExec")
+	beego.Router("/api/retentions/:id/executions/:eid", &retentionCtl.RetentionAPI{}, "get:GetRetentionExec")
+	beego.Router("/api/retentions/:id/executions", &retentionCtl.RetentionAPI{}, "get:ListRetentionExec")
+	beego.Router("/api/retentions/:id/executions/:eid/histories", &retentionCtl.RetentionAPI{}, "get:ListRetentionExecHistory")
+
 	beego.Router("/v2/*", &controllers.RegistryProxy{}, "*:Handle")
 
 	// APIs for chart repository

src/pkg/retention/controllers/retention.go

@@ -0,0 +1,159 @@
+package controllers
+
+import (
+	"github.com/goharbor/harbor/src/core/api"
+	"github.com/goharbor/harbor/src/pkg/retention"
+	"github.com/goharbor/harbor/src/pkg/retention/policy"
+	"github.com/goharbor/harbor/src/pkg/retention/q"
+	"time"
+)
+
+type RetentionAPI struct {
+	api.BaseController
+	manager retention.Manager
+}
+
+// Prepare validates the user
+func (t *RetentionAPI) Prepare() {
+	t.BaseController.Prepare()
+	t.manager = retention.NewManager()
+}
+
+func (r *RetentionAPI) GetRetention() {
+	id, err := r.GetIDFromURL()
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	p, err := r.manager.GetPolicy(id)
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	r.Data["json"] = p
+	r.ServeJSON()
+}
+
+func (r *RetentionAPI) CreateRetention() {
+	p := &policy.Metadata{}
+	isValid, err := r.DecodeJSONReqAndValidate(p)
+	if !isValid {
+		r.SendBadRequestError(err)
+		return
+	}
+
+	r.manager.CreatePolicy(p)
+}
+
+func (r *RetentionAPI) UpdateRetention() {
+	id, err := r.GetIDFromURL()
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	p := &policy.Metadata{}
+	isValid, err := r.DecodeJSONReqAndValidate(p)
+	if !isValid {
+		r.SendBadRequestError(err)
+		return
+	}
+	p.ID = id
+	r.manager.UpdatePolicy(p)
+}
+
+func (r *RetentionAPI) DeleteRetention() {
+	id, err := r.GetIDFromURL()
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	r.manager.DeletePolicy(id)
+}
+
+func (r *RetentionAPI) TriggerRetentionExec() {
+	id, err := r.GetIDFromURL()
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	exec := &retention.Execution{
+		PolicyID:  id,
+		StartTime: time.Now(),
+		Status:    "Running",
+	}
+	r.manager.CreateExecution(exec)
+}
+
+func (r *RetentionAPI) OperateRetentionExec() {
+	eid, err := r.GetSpecialIDFromURL("eid")
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	exec := &retention.Execution{}
+	isValid, err := r.DecodeJSONReqAndValidate(exec)
+	if !isValid {
+		r.SendBadRequestError(err)
+		return
+	}
+	exec.ID = eid
+	r.manager.UpdateExecution(nil)
+}
+
+func (r *RetentionAPI) GetRetentionExec() {
+	eid, err := r.GetSpecialIDFromURL("eid")
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	exec, err := r.manager.GetExecution(eid)
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	r.Data["json"] = exec
+	r.ServeJSON()
+}
+
+func (r *RetentionAPI) ListRetentionExec() {
+	page, size, err := r.GetPaginationParams()
+	if err != nil {
+		r.SendInternalServerError(err)
+		return
+	}
+	query := &q.Query{
+		PageNumber: page,
+		PageSize:   size,
+	}
+	execs, err := r.manager.ListExecutions(query)
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	r.Data["json"] = execs
+	r.ServeJSON()
+}
+
+func (r *RetentionAPI) ListRetentionExecHistory() {
+	eid, err := r.GetSpecialIDFromURL("eid")
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	page, size, err := r.GetPaginationParams()
+	if err != nil {
+		r.SendInternalServerError(err)
+		return
+	}
+	query := &q.Query{
+		PageNumber: page,
+		PageSize:   size,
+	}
+	his, err := r.manager.ListHistories(eid, query)
+	if err != nil {
+		r.SendBadRequestError(err)
+		return
+	}
+	r.Data["json"] = his
+	r.ServeJSON()
+}

src/pkg/retention/dao/models/retention.go

@@ -0,0 +1,64 @@
+package models
+
+import (
+	"github.com/astaxie/beego/orm"
+	"time"
+)
+
+func init() {
+	orm.RegisterModel(
+		new (RetentionPolicy),
+		new (RetentionExecution),
+		new (RetentionTask),
+		new (RetentionScheduleJob),
+		)
+}
+
+type RetentionPolicy struct {
+	ID int64 `orm:"pk;auto;column(id)" json:"id"`
+	// 'system', 'project' and 'repository'
+	ScopeLevel      string
+	ScopeReference  int64
+	TriggerKind     string
+	// json format, include algorithm, rules, exclusions
+	Data       string
+	CreateTime time.Time
+	UpdateTime time.Time
+}
+
+type RetentionExecution struct {
+	ID         int64 `orm:"pk;auto;column(id)" json:"id"`
+	PolicyID   int64
+	Status     string
+	StatusText string
+	Dry        bool
+	// manual, scheduled
+	Trigger    string
+	Total      int
+	Succeed    int
+	Failed     int
+	InProgress int
+	Stopped    int
+	StartTime  time.Time
+	EndTime    time.Time
+}
+
+type RetentionTask struct {
+	ID          int64
+	ExecutionID int64
+	// image, chart
+	ResourceType string
+	Resource     string
+	Status       string
+	StartTime    time.Time
+	EndTime      time.Time
+}
+
+type RetentionScheduleJob struct {
+	ID         int64
+	Status     string
+	PolicyID   int64
+	JobID      int64
+	CreateTime time.Time
+	UpdateTime time.Time
+}

src/pkg/retention/dao/retention.go

@@ -0,0 +1,96 @@
+package dao
+
+import (
+	"github.com/goharbor/harbor/src/common/dao"
+	"github.com/goharbor/harbor/src/pkg/retention/dao/models"
+	"github.com/goharbor/harbor/src/pkg/retention/q"
+)
+
+func CreatePolicy(p *models.RetentionPolicy) (int64, error) {
+	o := dao.GetOrmer()
+	return o.Insert(p)
+}
+
+func UpdatePolicy(p *models.RetentionPolicy) error {
+	o := dao.GetOrmer()
+	_, err := o.Update(p)
+	return err
+}
+
+func DeletePolicy(id int64) error {
+	o := dao.GetOrmer()
+	_, err := o.Delete(&models.RetentionPolicy{
+		ID: id,
+	})
+	return err
+}
+
+func GetPolicy(id int64) (*models.RetentionPolicy, error) {
+	o := dao.GetOrmer()
+	p := &models.RetentionPolicy{
+		ID: id,
+	}
+	if err := o.Read(p); err != nil {
+		return nil, err
+	} else {
+		return p, nil
+	}
+}
+
+func CreateExecution(e *models.RetentionExecution) (int64, error) {
+	o := dao.GetOrmer()
+	return o.Insert(e)
+}
+
+func UpdateExecution(e *models.RetentionExecution) error {
+	o := dao.GetOrmer()
+	_, err := o.Update(e)
+	return err
+}
+
+func DeleteExecution(id int64) error {
+	o := dao.GetOrmer()
+	_, err := o.Delete(&models.RetentionExecution{
+		ID: id,
+	})
+	return err
+}
+
+func GetExecution(id int64) (*models.RetentionExecution, error) {
+	o := dao.GetOrmer()
+	e := &models.RetentionExecution{
+		ID: id,
+	}
+	if err := o.Read(e); err != nil {
+		return nil, err
+	} else {
+		return e, nil
+	}
+}
+
+func ListExecutions(query *q.Query) ([]*models.RetentionExecution, error) {
+	o := dao.GetOrmer()
+	qs := o.QueryTable(new(models.RetentionExecution))
+	qs.Limit(query.PageSize, (query.PageNumber-1)*query.PageSize)
+	var execs []*models.RetentionExecution
+	_, err := qs.All(&execs)
+	if err != nil {
+		return nil, err
+	} else {
+		return execs, nil
+	}
+}
+
+func ListExecHistories(executionID int64, query *q.Query) ([]*models.RetentionTask, error) {
+	o := dao.GetOrmer()
+	qs := o.QueryTable(new(models.RetentionTask))
+	qs.Filter("Execution_ID", executionID)
+	qs.Limit(query.PageSize, (query.PageNumber-1)*query.PageSize)
+	var tasks []*models.RetentionTask
+	_, err := qs.All(&tasks)
+	if err != nil {
+		return nil, err
+	} else {
+		return tasks, nil
+	}
+}

src/pkg/retention/dao/retention_test.go

@@ -0,0 +1,98 @@
+package dao
+
+import (
+	"encoding/json"
+	"github.com/goharbor/harbor/src/common/dao"
+	"github.com/goharbor/harbor/src/pkg/retention/policy"
+	"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/goharbor/harbor/src/pkg/retention/dao/models"
+)
+
+func TestMain(m *testing.M) {
+	dao.PrepareTestForPostgresSQL()
+	os.Exit(m.Run())
+}
+
+func TestPolicy(t *testing.T) {
+	p := &policy.Metadata{
+		Algorithm: "OR",
+		Rules: []rule.Metadata{
+			{
+				ID:       1,
+				Priority: 1,
+				Template: "recentXdays",
+				Parameters: rule.Parameters{
+					"num": 10,
+				},
+				TagSelectors: []*rule.Selector{
+					{
+						Kind:       "label",
+						Decoration: "with",
+						Pattern:    "latest",
+					},
+					{
+						Kind:       "regularExpression",
+						Decoration: "matches",
+						Pattern:    "release-[\\d\\.]+",
+					},
+				},
+				ScopeSelectors: map[string][]*rule.Selector{
+					"repository": {
+						{
+							Kind:       "regularExpression",
+							Decoration: "matches",
+							Pattern:    ".+",
+						},
+					},
+				},
+			},
+		},
+		Trigger: &policy.Trigger{
+			Kind: "Schedule",
+			Settings: map[string]interface{}{
+				"cron": "* 22 11 * * *",
+			},
+		},
+		Scope: &policy.Scope{
+			Level:     "project",
+			Reference: 1,
+		},
+	}
+	p1 := &models.RetentionPolicy{
+		ScopeLevel:  p.Scope.Level,
+		TriggerKind: p.Trigger.Kind,
+		CreateTime:  time.Now(),
+		UpdateTime:  time.Now(),
+	}
+	data, _ := json.Marshal(p)
+	p1.Data = string(data)
+
+	id, err := CreatePolicy(p1)
+	assert.Nil(t, err)
+	assert.True(t, id > 0)
+
+	p1, err = GetPolicy(id)
+	assert.Nil(t, err)
+	assert.EqualValues(t, "project", p1.ScopeLevel)
+	assert.True(t, p1.ID > 0)
+
+	p1.ScopeLevel = "test"
+	err = UpdatePolicy(p1)
+	assert.Nil(t, err)
+	p1, err = GetPolicy(id)
+	assert.Nil(t, err)
+	assert.EqualValues(t, "test", p1.ScopeLevel)
+
+	err = DeletePolicy(id)
+	assert.Nil(t, err)
+
+	p1, err = GetPolicy(id)
+	assert.NotNil(t, err)
+	assert.True(t, strings.Contains(err.Error(), "no row found"))
+}

src/pkg/retention/manager.go

@@ -15,32 +15,145 @@
 package retention
 
 import (
+	"encoding/json"
+	"github.com/goharbor/harbor/src/pkg/retention/dao"
+	"github.com/goharbor/harbor/src/pkg/retention/dao/models"
 	"github.com/goharbor/harbor/src/pkg/retention/policy"
 	"github.com/goharbor/harbor/src/pkg/retention/q"
+	"time"
 )
 
 // Manager defines operations of managing policy
 type Manager interface {
 	// Create new policy and return uuid
-	CreatePolicy(p *policy.Metadata) (string, error)
+	CreatePolicy(p *policy.Metadata) (int64, error)
 	// Update the existing policy
 	// Full update
 	UpdatePolicy(p *policy.Metadata) error
 	// Delete the specified policy
 	// No actual use so far
-	DeletePolicy(ID string) error
+	DeletePolicy(ID int64) error
 	// Get the specified policy
-	GetPolicy(ID string) (*policy.Metadata, error)
+	GetPolicy(ID int64) (*policy.Metadata, error)
 	// Create a new retention execution
-	CreateExecution(execution *Execution) (string, error)
+	CreateExecution(execution *Execution) (int64, error)
 	// Update the specified execution
 	UpdateExecution(execution *Execution) error
 	// Get the specified execution
-	GetExecution(eid string) (*Execution, error)
+	GetExecution(eid int64) (*Execution, error)
 	// List execution histories
 	ListExecutions(query *q.Query) ([]*Execution, error)
 	// Add new history
 	AppendHistory(history *History) error
 	// List all the histories marked by the specified execution
-	ListHistories(executionID string, query *q.Query) ([]*History, error)
+	ListHistories(executionID int64, query *q.Query) ([]*History, error)
+}
+
+type DefaultManager struct {
+}
+
+func (d *DefaultManager) CreatePolicy(p *policy.Metadata) (int64, error) {
+	var p1 *models.RetentionPolicy
+	p1.ScopeLevel = p.Scope.Level
+	p1.TriggerKind = p.Trigger.Kind
+	data, _ := json.Marshal(p)
+	p1.Data = string(data)
+	p1.CreateTime = time.Now()
+	p1.UpdateTime = p1.CreateTime
+	return dao.CreatePolicy(p1)
+}
+
+func (d *DefaultManager) UpdatePolicy(p *policy.Metadata) error {
+	var p1 *models.RetentionPolicy
+	p1.ID = p.ID
+	p1.ScopeLevel = p.Scope.Level
+	p1.TriggerKind = p.Trigger.Kind
+	p.ID = 0
+	data, _ := json.Marshal(p)
+	p.ID = p1.ID
+	p1.Data = string(data)
+	p1.UpdateTime = time.Now()
+	return dao.UpdatePolicy(p1)
+}
+
+func (d *DefaultManager) DeletePolicy(id int64) error {
+	return dao.DeletePolicy(id)
+}
+
+func (d *DefaultManager) GetPolicy(id int64) (*policy.Metadata, error) {
+	if p1,err:=dao.GetPolicy(id);err!=nil{
+		return nil,err
+	}else{
+		var p *policy.Metadata
+		if err=json.Unmarshal([]byte(p1.Data), p);err!=nil{
+			return nil,err
+		}else{
+			return p,nil
+		}
+	}
+}
+
+func (d *DefaultManager) CreateExecution(execution *Execution) (int64, error) {
+	var exec *models.RetentionExecution
+	exec.PolicyID=execution.PolicyID
+	exec.StartTime=time.Now()
+	exec.Status="Running"
+	return dao.CreateExecution(exec)
+}
+
+func (d *DefaultManager) UpdateExecution(execution *Execution) error {
+	var exec *models.RetentionExecution
+	exec.ID = execution.ID
+	exec.PolicyID=execution.PolicyID
+	exec.StartTime=time.Now()
+	exec.Status="Running"
+	return dao.UpdateExecution(exec)
+}
+
+func (d *DefaultManager) ListExecutions(query *q.Query) ([]*Execution, error) {
+	return []*Execution{
+		{
+			ID:        1,
+			PolicyID:  1,
+			StartTime: time.Now().Add(-time.Minute),
+			EndTime:   time.Now(),
+			Status:    "Success",
+		},
+		{
+			ID:        2,
+			PolicyID:  1,
+			StartTime: time.Now().Add(-time.Minute),
+			EndTime:   time.Now(),
+			Status:    "Failed",
+		},
+		{
+			ID:        3,
+			PolicyID:  1,
+			StartTime: time.Now().Add(-time.Minute),
+			EndTime:   time.Now(),
+			Status:    "Running",
+		},
+	}, nil
+}
+
+func (d *DefaultManager) GetExecution(eid int64) (*Execution, error) {
+	return &Execution{
+		ID:        1,
+		PolicyID:  1,
+		StartTime: time.Now().Add(-time.Minute),
+		EndTime:   time.Now(),
+		Status:    "Success",
+	}, nil
+}
+
+func (d *DefaultManager) ListHistories(executionID int64, query *q.Query) ([]*History, error) {
+	panic("implement me")
+}
+
+func (d *DefaultManager) AppendHistory(history *History) error {
+	panic("implement me")
+}
+
+func NewManager() Manager {
+	return &DefaultManager{}
 }

src/pkg/retention/models.go

@@ -18,10 +18,10 @@ import "time"
 
 // Execution of retention
 type Execution struct {
-	ID        string    `json:"id"`
-	PolicyID  string    `json:"policy_id"`
+	ID        int64       `json:"id,omitempty"`
+	PolicyID  int64     `json:"policy_id"`
 	StartTime time.Time `json:"start_time"`
-	EndTime   time.Time `json:"end_time"`
+	EndTime   time.Time `json:"end_time,omitempty"`
 	Status    string    `json:"status"`
 }
 
@@ -37,7 +37,7 @@ type TaskSubmitResult struct {
 type History struct {
 	ExecutionID string `json:"execution_id"`
 	Rule        struct {
-		ID          string `json:"id"`
+		ID          int    `json:"id"`
 		DisplayText string `json:"display_text"`
 	} `json:"rule_id"`
 	// full path: :ns/:repo:tag

src/pkg/retention/policy/models.go

@@ -26,7 +26,7 @@ const (
 // Metadata of policy
 type Metadata struct {
 	// UUID of the policy
-	ID string `json:"id"`
+	ID int64 `json:"id,omitempty"`
 
 	// Algorithm applied to the rules
 	// "OR" / "AND"

src/pkg/retention/policy/rule/models.go

@@ -17,7 +17,7 @@ package rule
 // Metadata of the retention rule
 type Metadata struct {
 	// UUID of rule
-	ID string `json:"id"`
+	ID int `json:"id"`
 
 	// Priority of rule when doing calculating
 	Priority int `json:"priority"`

src/pkg/retention/q/query.go

@@ -16,6 +16,6 @@ package q
 
 // Query parameters
 type Query struct {
-	PageNumber int
-	PageSize   int
+	PageNumber int64
+	PageSize   int64
 }