Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-20 07:37:38 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #8296 from bitsf/tag_retention_controller

Browse Source 
Tag retention controller
This commit is contained in:
Steven Zou 2019-07-16 22:45:36 +08:00 committed by GitHub
commit 7a1e955085
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 362 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
make/migrations/postgresql/0010_1.9.0_schema.up.sql
View File

@ -7,4 +7,52 @@ CREATE TABLE cve_whitelist (
expires_at bigint,
items text NOT NULL,
UNIQUE (project_id)
);
);
create table retention_policy
(
id serial PRIMARY KEY NOT NULL,
scope_level varchar(20),
scope_reference integer,
trigger_kind varchar(20),
data text,
create_time time,
update_time time
);
create table retention_execution
(
id integer PRIMARY KEY NOT NULL,
policy_id integer,
status varchar(20),
status_text text,
dry boolean,
trigger varchar(20),
total integer,
succeed integer,
failed integer,
in_progress integer,
stopped integer,
start_time time,
end_time time
);
create table retention_task
(
id integer PRIMARY KEY NOT NULL,
execution_id integer,
rule_id integer,
rule_display_text varchar(255),
artifact varchar(255),
timestamp time
);
create table retention_schedule_job
(
id integer PRIMARY KEY NOT NULL,
status varchar(20),
policy_id integer,
job_id integer,
create_time time,
update_time time
);

15
src/common/api/base.go
View File

@ -131,6 +131,21 @@ func (b *BaseAPI) GetIDFromURL() (int64, error) {
return id, nil
}
// GetSpecialIDFromURL checks the ID with special name in request URL
func (b *BaseAPI) GetSpecialIDFromURL(name string) (int64, error) {
idStr := b.Ctx.Input.Param(":" + name)
if len(idStr) == 0 {
return 0, fmt.Errorf("invalid %s in URL", name)
}
id, err := strconv.ParseInt(idStr, 10, 64)
if err != nil || id <= 0 {
return 0, errors.New("invalid ID in URL")
}
return id, nil
}
// SetPaginationHeader set"Link" and "X-Total-Count" header for pagination request
func (b *BaseAPI) SetPaginationHeader(total, page, pageSize int64) {
b.Ctx.ResponseWriter.Header().Set("X-Total-Count", strconv.FormatInt(total, 10))

4
src/core/router.go
View File

@ -144,8 +144,8 @@ func initRouters() {
beego.Router("/api/retentions", &retentionCtl.RetentionAPI{}, "post:CreateRetention")
beego.Router("/api/retentions/:id", &retentionCtl.RetentionAPI{}, "put:UpdateRetention")
beego.Router("/api/retentions/:id", &retentionCtl.RetentionAPI{}, "delete:DeleteRetention")
beego.Router("/api/retentions/:id/executions", &retentionCtl.RetentionAPI{}, "delete:TriggerRetentionExec")
beego.Router("/api/retentions/:id/executions/:eid", &retentionCtl.RetentionAPI{}, "delete:OperateRetentionExec")
beego.Router("/api/retentions/:id/executions", &retentionCtl.RetentionAPI{}, "post:TriggerRetentionExec")
beego.Router("/api/retentions/:id/executions/:eid", &retentionCtl.RetentionAPI{}, "put:OperateRetentionExec")
beego.Router("/api/retentions/:id/executions/:eid", &retentionCtl.RetentionAPI{}, "get:GetRetentionExec")
beego.Router("/api/retentions/:id/executions", &retentionCtl.RetentionAPI{}, "get:ListRetentionExec")
beego.Router("/api/retentions/:id/executions/:eid/histories", &retentionCtl.RetentionAPI{}, "get:ListRetentionExecHistory")

115
src/pkg/retention/controllers/retention.go
View File

@ -3,19 +3,24 @@ package controllers
import (
"github.com/goharbor/harbor/src/core/api"
"github.com/goharbor/harbor/src/pkg/retention"
"github.com/goharbor/harbor/src/pkg/retention/policy"
"github.com/goharbor/harbor/src/pkg/retention/q"
"time"
)
// RetentionAPI ...
type RetentionAPI struct {
api.BaseController
manager retention.Manager
}
// Prepare validates the user
func (t *RetentionAPI) Prepare() {
t.BaseController.Prepare()
t.manager = retention.NewManager()
func (r *RetentionAPI) Prepare() {
r.BaseController.Prepare()
r.manager = retention.NewManager()
}
// GetRetention Get Retention
func (r *RetentionAPI) GetRetention() {
id, err := r.GetIDFromURL()
if err != nil {
@ -31,19 +36,36 @@ func (r *RetentionAPI) GetRetention() {
r.ServeJSON()
}
// CreateRetention Create Retention
func (r *RetentionAPI) CreateRetention() {
r.manager.CreatePolicy(nil)
p := &policy.Metadata{}
isValid, err := r.DecodeJSONReqAndValidate(p)
if !isValid {
r.SendBadRequestError(err)
return
}
r.manager.CreatePolicy(p)
}
// UpdateRetention Update Retention
func (r *RetentionAPI) UpdateRetention() {
_, err := r.GetIDFromURL()
id, err := r.GetIDFromURL()
if err != nil {
r.SendBadRequestError(err)
return
}
r.manager.UpdatePolicy(nil)
p := &policy.Metadata{}
isValid, err := r.DecodeJSONReqAndValidate(p)
if !isValid {
r.SendBadRequestError(err)
return
}
p.ID = id
r.manager.UpdatePolicy(p)
}
// DeleteRetention Delete Retention
func (r *RetentionAPI) DeleteRetention() {
id, err := r.GetIDFromURL()
if err != nil {
@ -53,22 +75,95 @@ func (r *RetentionAPI) DeleteRetention() {
r.manager.DeletePolicy(id)
}
// TriggerRetentionExec Trigger Retention Execution
func (r *RetentionAPI) TriggerRetentionExec() {
//r.manager.CreateExecution(nil)
id, err := r.GetIDFromURL()
if err != nil {
r.SendBadRequestError(err)
return
}
exec := &retention.Execution{
PolicyID: id,
StartTime: time.Now(),
Status: "Running",
}
r.manager.CreateExecution(exec)
}
// OperateRetentionExec Operate Retention Execution
func (r *RetentionAPI) OperateRetentionExec() {
eid, err := r.GetSpecialIDFromURL("eid")
if err != nil {
r.SendBadRequestError(err)
return
}
exec := &retention.Execution{}
isValid, err := r.DecodeJSONReqAndValidate(exec)
if !isValid {
r.SendBadRequestError(err)
return
}
exec.ID = eid
r.manager.UpdateExecution(nil)
}
// GetRetentionExec Get Retention Execution
func (r *RetentionAPI) GetRetentionExec() {
//r.manager.GetExecution(eid)
eid, err := r.GetSpecialIDFromURL("eid")
if err != nil {
r.SendBadRequestError(err)
return
}
exec, err := r.manager.GetExecution(eid)
if err != nil {
r.SendBadRequestError(err)
return
}
r.Data["json"] = exec
r.ServeJSON()
}
// ListRetentionExec List Retention Execution
func (r *RetentionAPI) ListRetentionExec() {
r.manager.ListExecutions(nil)
page, size, err := r.GetPaginationParams()
if err != nil {
r.SendInternalServerError(err)
return
}
query := &q.Query{
PageNumber: page,
PageSize: size,
}
execs, err := r.manager.ListExecutions(query)
if err != nil {
r.SendBadRequestError(err)
return
}
r.Data["json"] = execs
r.ServeJSON()
}
// ListRetentionExecHistory List Retention Execution Histories
func (r *RetentionAPI) ListRetentionExecHistory() {
//r.manager.ListHistories(eid, nil)
eid, err := r.GetSpecialIDFromURL("eid")
if err != nil {
r.SendBadRequestError(err)
return
}
page, size, err := r.GetPaginationParams()
if err != nil {
r.SendInternalServerError(err)
return
}
query := &q.Query{
PageNumber: page,
PageSize: size,
}
his, err := r.manager.ListHistories(eid, query)
if err != nil {
r.SendBadRequestError(err)
return
}
r.Data["json"] = his
r.ServeJSON()
}

34
src/pkg/retention/dao/models/retention.go
View File

@ -7,25 +7,27 @@ import (
func init() {
orm.RegisterModel(
new (RetentionPolicy),
new (RetentionExecution),
new (RetentionTask),
new (RetentionScheduleJob),
)
new(RetentionPolicy),
new(RetentionExecution),
new(RetentionTask),
new(RetentionScheduleJob),
)
}
// RetentionPolicy Retention Policy
type RetentionPolicy struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
// 'system', 'project' and 'repository'
ScopeLevel string
ScopeReference int64
TriggerKind string
ScopeLevel string
ScopeReference int64
TriggerKind string
// json format, include algorithm, rules, exclusions
Data string
CreateTime time.Time
UpdateTime time.Time
}
// RetentionExecution Retention Execution
type RetentionExecution struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
PolicyID int64
@ -43,17 +45,17 @@ type RetentionExecution struct {
EndTime time.Time
}
// RetentionTask Retention Task
type RetentionTask struct {
ID int64
ExecutionID int64
// image, chart
ResourceType string
Resource string
Status string
StartTime time.Time
EndTime time.Time
ID int64
ExecutionID int64
RuleID int
RuleDisplayText string
Artifact string
Timestamp time.Time
}
// RetentionScheduleJob Retention Schedule Job
type RetentionScheduleJob struct {
ID int64
Status string

75
src/pkg/retention/dao/retention.go
View File

@ -3,19 +3,23 @@ package dao
import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/pkg/retention/dao/models"
"github.com/goharbor/harbor/src/pkg/retention/q"
)
// CreatePolicy Create Policy
func CreatePolicy(p *models.RetentionPolicy) (int64, error) {
o := dao.GetOrmer()
return o.Insert(p)
}
// UpdatePolicy Update Policy
func UpdatePolicy(p *models.RetentionPolicy) error {
o := dao.GetOrmer()
_, err := o.Update(p)
return err
}
// DeletePolicy Delete Policy
func DeletePolicy(id int64) error {
o := dao.GetOrmer()
_, err := o.Delete(&models.RetentionPolicy{
@ -24,6 +28,7 @@ func DeletePolicy(id int64) error {
return err
}
// GetPolicy Get Policy
func GetPolicy(id int64) (*models.RetentionPolicy, error) {
o := dao.GetOrmer()
p := &models.RetentionPolicy{
@ -31,7 +36,73 @@ func GetPolicy(id int64) (*models.RetentionPolicy, error) {
}
if err := o.Read(p); err != nil {
return nil, err
} else {
return p, nil
}
return p, nil
}
// CreateExecution Create Execution
func CreateExecution(e *models.RetentionExecution) (int64, error) {
o := dao.GetOrmer()
return o.Insert(e)
}
// UpdateExecution Update Execution
func UpdateExecution(e *models.RetentionExecution) error {
o := dao.GetOrmer()
_, err := o.Update(e)
return err
}
// DeleteExecution Delete Execution
func DeleteExecution(id int64) error {
o := dao.GetOrmer()
_, err := o.Delete(&models.RetentionExecution{
ID: id,
})
return err
}
// GetExecution Get Execution
func GetExecution(id int64) (*models.RetentionExecution, error) {
o := dao.GetOrmer()
e := &models.RetentionExecution{
ID: id,
}
if err := o.Read(e); err != nil {
return nil, err
}
return e, nil
}
// ListExecutions List Executions
func ListExecutions(query *q.Query) ([]*models.RetentionExecution, error) {
o := dao.GetOrmer()
qs := o.QueryTable(new(models.RetentionExecution))
qs.Limit(query.PageSize, (query.PageNumber-1)*query.PageSize)
var execs []*models.RetentionExecution
_, err := qs.All(&execs)
if err != nil {
return nil, err
}
return execs, nil
}
// ListExecHistories List Execution Histories
func ListExecHistories(executionID int64, query *q.Query) ([]*models.RetentionTask, error) {
o := dao.GetOrmer()
qs := o.QueryTable(new(models.RetentionTask))
qs.Filter("Execution_ID", executionID)
qs.Limit(query.PageSize, (query.PageNumber-1)*query.PageSize)
var tasks []*models.RetentionTask
_, err := qs.All(&tasks)
if err != nil {
return nil, err
}
return tasks, nil
}
// AppendExecHistory Append Execution History
func AppendExecHistory(t *models.RetentionTask) (int64, error) {
o := dao.GetOrmer()
return o.Insert(t)
}

14
src/pkg/retention/dao/retention_test.go
View File

@ -21,7 +21,6 @@ func TestMain(m *testing.M) {
func TestPolicy(t *testing.T) {
p := &policy.Metadata{
ID: 1,
Algorithm: "OR",
Rules: []rule.Metadata{
{
@ -43,11 +42,13 @@ func TestPolicy(t *testing.T) {
Pattern: "release-[\\d\\.]+",
},
},
ScopeSelectors: []*rule.Selector{
{
Kind: "regularExpression",
Decoration: "matches",
Pattern: ".+",
ScopeSelectors: map[string][]*rule.Selector{
"repository": {
{
Kind: "regularExpression",
Decoration: "matches",
Pattern: ".+",
},
},
},
},
@ -79,6 +80,7 @@ func TestPolicy(t *testing.T) {
p1, err = GetPolicy(id)
assert.Nil(t, err)
assert.EqualValues(t, "project", p1.ScopeLevel)
assert.True(t, p1.ID > 0)
p1.ScopeLevel = "test"
err = UpdatePolicy(p1)

131
src/pkg/retention/manager.go
View File

@ -36,7 +36,7 @@ type Manager interface {
// Get the specified policy
GetPolicy(ID int64) (*policy.Metadata, error)
// Create a new retention execution
CreateExecution(execution *Execution) (string, error)
CreateExecution(execution *Execution) (int64, error)
// Update the specified execution
UpdateExecution(execution *Execution) error
// Get the specified execution
@ -49,9 +49,11 @@ type Manager interface {
ListHistories(executionID int64, query *q.Query) ([]*History, error)
}
// DefaultManager ...
type DefaultManager struct {
}
// CreatePolicy Create Policy
func (d *DefaultManager) CreatePolicy(p *policy.Metadata) (int64, error) {
var p1 *models.RetentionPolicy
p1.ScopeLevel = p.Scope.Level
@ -63,86 +65,123 @@ func (d *DefaultManager) CreatePolicy(p *policy.Metadata) (int64, error) {
return dao.CreatePolicy(p1)
}
// UpdatePolicy Update Policy
func (d *DefaultManager) UpdatePolicy(p *policy.Metadata) error {
var p1 *models.RetentionPolicy
p1.ID = p.ID
p1.ScopeLevel = p.Scope.Level
p1.TriggerKind = p.Trigger.Kind
p.ID = 0
data, _ := json.Marshal(p)
p.ID = p1.ID
p1.Data = string(data)
p1.UpdateTime = time.Now()
return dao.UpdatePolicy(p1)
}
// DeletePolicy Delete Policy
func (d *DefaultManager) DeletePolicy(id int64) error {
return dao.DeletePolicy(id)
}
// GetPolicy Get Policy
func (d *DefaultManager) GetPolicy(id int64) (*policy.Metadata, error) {
if p1,err:=dao.GetPolicy(id);err!=nil{
return nil,err
}else{
var p *policy.Metadata
if err=json.Unmarshal([]byte(p1.Data), p);err!=nil{
return nil,err
}else{
return p,nil
}
p1, err := dao.GetPolicy(id)
if err != nil {
return nil, err
}
var p *policy.Metadata
if err = json.Unmarshal([]byte(p1.Data), p); err != nil {
return nil, err
}
return p, nil
}
func (d *DefaultManager) CreateExecution(execution *Execution) (string, error) {
panic("implement me")
// CreateExecution Create Execution
func (d *DefaultManager) CreateExecution(execution *Execution) (int64, error) {
var exec *models.RetentionExecution
exec.PolicyID = execution.PolicyID
exec.StartTime = time.Now()
exec.Status = "Running"
return dao.CreateExecution(exec)
}
// UpdateExecution Update Execution
func (d *DefaultManager) UpdateExecution(execution *Execution) error {
panic("implement me")
var exec *models.RetentionExecution
exec.ID = execution.ID
exec.PolicyID = execution.PolicyID
exec.StartTime = time.Now()
exec.Status = "Running"
return dao.UpdateExecution(exec)
}
// ListExecutions List Executions
func (d *DefaultManager) ListExecutions(query *q.Query) ([]*Execution, error) {
return []*Execution{
{
ID: 1,
PolicyID: 1,
StartTime: time.Now().Add(-time.Minute),
EndTime: time.Now(),
Status: "Success",
},
{
ID: 2,
PolicyID: 1,
StartTime: time.Now().Add(-time.Minute),
EndTime: time.Now(),
Status: "Failed",
},
{
ID: 3,
PolicyID: 1,
StartTime: time.Now().Add(-time.Minute),
EndTime: time.Now(),
Status: "Running",
},
}, nil
execs, err := dao.ListExecutions(query)
if err != nil {
return nil, err
}
var execs1 []*Execution
for _, e := range execs {
var e1 *Execution
e1.ID = e.ID
e1.PolicyID = e.PolicyID
e1.Status = e.Status
e1.StartTime = e.StartTime
e1.EndTime = e.EndTime
execs1 = append(execs1, e1)
}
return execs1, nil
}
// GetExecution Get Execution
func (d *DefaultManager) GetExecution(eid int64) (*Execution, error) {
return &Execution{
ID: 1,
PolicyID: 1,
StartTime: time.Now().Add(-time.Minute),
EndTime: time.Now(),
Status: "Success",
}, nil
e, err := dao.GetExecution(eid)
if err != nil {
return nil, err
}
var e1 *Execution
e1.ID = e.ID
e1.PolicyID = e.PolicyID
e1.Status = e.Status
e1.StartTime = e.StartTime
e1.EndTime = e.EndTime
return e1, nil
}
// ListHistories List Histories
func (d *DefaultManager) ListHistories(executionID int64, query *q.Query) ([]*History, error) {
panic("implement me")
his, err := dao.ListExecHistories(executionID, query)
if err != nil {
return nil, err
}
var his1 []*History
for _, h := range his {
var h1 *History
h1.ExecutionID = h.ExecutionID
h1.Artifact = h.Artifact
h1.Rule.ID = h.RuleID
h1.Rule.DisplayText = h.RuleDisplayText
h1.Timestamp = h.Timestamp
his1 = append(his1, h1)
}
return his1, nil
}
func (d *DefaultManager) AppendHistory(history *History) error {
panic("implement me")
// AppendHistory Append History
func (d *DefaultManager) AppendHistory(h *History) error {
var h1 *models.RetentionTask
h1.ExecutionID = h.ExecutionID
h1.Artifact = h.Artifact
h1.RuleID = h.Rule.ID
h1.RuleDisplayText = h.Rule.DisplayText
h1.Timestamp = h.Timestamp
_, err := dao.AppendExecHistory(h1)
return err
}
// NewManager ...
func NewManager() Manager {
return &DefaultManager{}
}

8
src/pkg/retention/models.go
View File

@ -18,10 +18,10 @@ import "time"
// Execution of retention
type Execution struct {
ID int `json:"id"`
PolicyID int `json:"policy_id"`
ID int64 `json:"id,omitempty"`
PolicyID int64 `json:"policy_id"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
EndTime time.Time `json:"end_time,omitempty"`
Status string `json:"status"`
}
@ -35,7 +35,7 @@ type TaskSubmitResult struct {
// History of retention
type History struct {
ExecutionID string `json:"execution_id"`
ExecutionID int64 `json:"execution_id"`
Rule struct {
ID int `json:"id"`
DisplayText string `json:"display_text"`

2
src/pkg/retention/policy/models.go
View File

@ -26,7 +26,7 @@ const (
// Metadata of policy
type Metadata struct {
// UUID of the policy
ID int64 `json:"-"`
ID int64 `json:"id,omitempty"`
// Algorithm applied to the rules
// "OR" / "AND"

4
src/pkg/retention/q/query.go
View File

@ -16,6 +16,6 @@ package q
// Query parameters
type Query struct {
PageNumber int
PageSize int
PageNumber int64
PageSize int64
}