Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

update code per review comments 

Signed-off-by: wang yan <wangyan@vmware.com>
This commit is contained in:
wang yan 2020-09-23 16:18:11 +08:00 committed by Wang Yan
parent dba5522d0b
commit 1bb79d402d
13 changed files with 512 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
api/v2.0/swagger.yaml
View File

@ -2079,6 +2079,8 @@ paths:
- $ref: '#/parameters/gcId'
tags:
- gc
produces:
- text/plain
responses:
'200':
description: Get successfully.
@ -2098,7 +2100,7 @@ paths:
get:
summary: Get gc's schedule.
description: This endpoint is for get schedule of gc job.
operationId: getSchedule
operationId: getGCSchedule
tags:
- gc
responses:
@ -2116,7 +2118,7 @@ paths:
summary: Create a gc schedule.
description: |
This endpoint is for update gc schedule.
operationId: postSchedule
operationId: createGCSchedule
parameters:
- name: schedule
in: body
@ -2143,7 +2145,7 @@ paths:
summary: Update gc's schedule.
description: |
This endpoint is for update gc schedule.
operationId: putSchedule
operationId: updateGCSchedule
parameters:
- name: schedule
in: body

2
make/migrations/postgresql/0050_2.2.0_schema.up.sql
View File

@ -270,4 +270,4 @@ BEGIN
END $$;
ALTER TABLE execution ALTER COLUMN vendor_type TYPE varchar(64);
ALTER TABLE schedule ALTER COLUMN vendor_type TYPE varchar(64);
ALTER TABLE task ALTER COLUMN vendor_type TYPE varchar(64);

206
src/controller/gc/controller.go
View File

@ -2,43 +2,48 @@ package gc
import (
"context"
"encoding/json"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
)
var (
// GCCtl is a global garbage collection controller instance
GCCtl = NewController()
// Ctl is a global garbage collection controller instance
Ctl = NewController()
)
const (
// SchedulerCallback ...
SchedulerCallback = "GARBAGE_COLLECTION"
// gcVendorType ...
gcVendorType = "GARBAGE_COLLECTION"
// GCVendorType ...
GCVendorType = "GARBAGE_COLLECTION"
)
// Controller manages the tags
type Controller interface {
// Start start a manual gc job
Start(ctx context.Context, parameters map[string]interface{}) error
Start(ctx context.Context, policy Policy, trigger string) (int64, error)
// Stop stop a gc job
Stop(ctx context.Context, taskID int64) error
// GetLog get the gc log by id
GetLog(ctx context.Context, id int64) ([]byte, error)
// History list all of gc executions
History(ctx context.Context, query *q.Query) ([]*History, error)
// Count count the gc executions
Count(ctx context.Context, query *q.Query) (int64, error)
Stop(ctx context.Context, id int64) error
// ExecutionCount returns the total count of executions according to the query
ExecutionCount(ctx context.Context, query *q.Query) (count int64, err error)
// ListExecutions lists the executions according to the query
ListExecutions(ctx context.Context, query *q.Query) (executions []*Execution, err error)
// GetExecution gets the specific execution
GetExecution(ctx context.Context, executionID int64) (execution *Execution, err error)
// GetTask gets the specific task
GetTask(ctx context.Context, id int64) (*Task, error)
// GetTaskLog gets log of the specific task
GetTaskLog(ctx context.Context, id int64) ([]byte, error)
// GetSchedule get the current gc schedule
GetSchedule(ctx context.Context) (*scheduler.Schedule, error)
// CreateSchedule create the gc schedule with cron string
CreateSchedule(ctx context.Context, cron string, parameters map[string]interface{}) (int64, error)
// CreateSchedule create the gc schedule with cron type & string
CreateSchedule(ctx context.Context, cronType, cron string, policy Policy) (int64, error)
// DeleteSchedule remove the gc schedule
DeleteSchedule(ctx context.Context) error
}
@ -59,97 +64,105 @@ type controller struct {
}
// Start starts the manual GC
func (c *controller) Start(ctx context.Context, parameters map[string]interface{}) error {
execID, err := c.exeMgr.Create(ctx, gcVendorType, -1, task.ExecutionTriggerManual, parameters)
func (c *controller) Start(ctx context.Context, policy Policy, trigger string) (int64, error) {
para := make(map[string]interface{})
para["delete_untagged"] = policy.DeleteUntagged
para["dry_run"] = policy.DryRun
para["redis_url_reg"] = policy.ExtraAttrs["redis_url_reg"]
para["time_window"] = policy.ExtraAttrs["time_window"]
execID, err := c.exeMgr.Create(ctx, GCVendorType, -1, trigger, para)
if err != nil {
return err
return -1, err
}
taskID, err := c.taskMgr.Create(ctx, execID, &task.Job{
_, err = c.taskMgr.Create(ctx, execID, &task.Job{
Name: job.ImageGC,
Metadata: &job.Metadata{
JobKind: job.KindGeneric,
},
Parameters: parameters,
Parameters: para,
})
if err != nil {
return err
return -1, err
}
defer func() {
if err == nil {
return
}
if err := c.taskMgr.Stop(ctx, taskID); err != nil {
log.Errorf("failed to stop the task %d: %v", taskID, err)
}
}()
return nil
return execID, nil
}
// Stop ...
func (c *controller) Stop(ctx context.Context, taskID int64) error {
return c.taskMgr.Stop(ctx, taskID)
func (c *controller) Stop(ctx context.Context, id int64) error {
return c.exeMgr.Stop(ctx, id)
}
// GetLog ...
func (c *controller) GetLog(ctx context.Context, executionID int64) ([]byte, error) {
tasks, err := c.taskMgr.List(ctx, q.New(q.KeyWords{"ExecutionID": executionID}))
// ExecutionCount ...
func (c *controller) ExecutionCount(ctx context.Context, query *q.Query) (int64, error) {
query.Keywords["VendorType"] = GCVendorType
return c.exeMgr.Count(ctx, query)
}
// ListExecutions ...
func (c *controller) ListExecutions(ctx context.Context, query *q.Query) ([]*Execution, error) {
query = q.MustClone(query)
query.Keywords["VendorType"] = GCVendorType
execs, err := c.exeMgr.List(ctx, query)
if err != nil {
return nil, err
}
var executions []*Execution
for _, exec := range execs {
executions = append(executions, convertExecution(exec))
}
return executions, nil
}
// GetExecution ...
func (c *controller) GetExecution(ctx context.Context, id int64) (*Execution, error) {
execs, err := c.exeMgr.List(ctx, &q.Query{
Keywords: map[string]interface{}{
"ID": id,
"VendorType": GCVendorType,
},
})
if err != nil {
return nil, err
}
if len(execs) == 0 {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).
WithMessage("garbage collection execution %d not found", id)
}
return convertExecution(execs[0]), nil
}
// GetTask ...
func (c *controller) GetTask(ctx context.Context, id int64) (*Task, error) {
tasks, err := c.taskMgr.List(ctx, &q.Query{
Keywords: map[string]interface{}{
"ID": id,
"VendorType": GCVendorType,
},
})
if err != nil {
return nil, err
}
if len(tasks) == 0 {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc task is found")
return nil, errors.New(nil).WithCode(errors.NotFoundCode).
WithMessage("garbage collection task %d not found", id)
}
return c.taskMgr.GetLog(ctx, tasks[0].ID)
return convertTask(tasks[0]), nil
}
// Count ...
func (c *controller) Count(ctx context.Context, query *q.Query) (int64, error) {
query.Keywords["VendorType"] = gcVendorType
return c.exeMgr.Count(ctx, query)
}
// History ...
func (c *controller) History(ctx context.Context, query *q.Query) ([]*History, error) {
var hs []*History
query.Keywords["VendorType"] = gcVendorType
exes, err := c.exeMgr.List(ctx, query)
// GetTaskLog ...
func (c *controller) GetTaskLog(ctx context.Context, id int64) ([]byte, error) {
_, err := c.GetTask(ctx, id)
if err != nil {
return nil, err
}
for _, exe := range exes {
tasks, err := c.taskMgr.List(ctx, q.New(q.KeyWords{"ExecutionID": exe.ID}))
if err != nil {
return nil, err
}
if len(tasks) == 0 {
continue
}
extraAttrsString, err := json.Marshal(exe.ExtraAttrs)
if err != nil {
return nil, err
}
hs = append(hs, &History{
ID: exe.ID,
Name: gcVendorType,
Kind: exe.Trigger,
Parameters: string(extraAttrsString),
Deleted: false,
Schedule: Schedule{Schedule: &ScheduleParam{
Type: exe.Trigger,
}},
Status: tasks[0].Status,
CreationTime: tasks[0].CreationTime,
UpdateTime: tasks[0].UpdateTime,
})
}
return hs, nil
return c.taskMgr.GetLog(ctx, id)
}
// GetSchedule ...
func (c *controller) GetSchedule(ctx context.Context) (*scheduler.Schedule, error) {
sch, err := c.schedulerMgr.ListSchedules(ctx, q.New(q.KeyWords{"VendorType": gcVendorType}))
sch, err := c.schedulerMgr.ListSchedules(ctx, q.New(q.KeyWords{"VendorType": GCVendorType}))
if err != nil {
return nil, err
}
@ -163,11 +176,40 @@ func (c *controller) GetSchedule(ctx context.Context) (*scheduler.Schedule, erro
}
// CreateSchedule ...
func (c *controller) CreateSchedule(ctx context.Context, cron string, parameters map[string]interface{}) (int64, error) {
return c.schedulerMgr.Schedule(ctx, gcVendorType, -1, cron, SchedulerCallback, parameters)
func (c *controller) CreateSchedule(ctx context.Context, cronType, cron string, policy Policy) (int64, error) {
return c.schedulerMgr.Schedule(ctx, GCVendorType, -1, cronType, cron, SchedulerCallback, policy)
}
// DeleteSchedule ...
func (c *controller) DeleteSchedule(ctx context.Context) error {
return c.schedulerMgr.UnScheduleByVendor(ctx, gcVendorType, -1)
return c.schedulerMgr.UnScheduleByVendor(ctx, GCVendorType, -1)
}
func convertExecution(exec *task.Execution) *Execution {
return &Execution{
ID: exec.ID,
Status: exec.Status,
StatusMessage: exec.StatusMessage,
Trigger: exec.Trigger,
ExtraAttrs: exec.ExtraAttrs,
StartTime: exec.StartTime,
EndTime: exec.EndTime,
}
}
func convertTask(task *task.Task) *Task {
return &Task{
ID: task.ID,
ExecutionID: task.ExecutionID,
Status: task.Status,
StatusMessage: task.StatusMessage,
RunCount: task.RunCount,
DeleteUntagged: task.GetBoolFromExtraAttrs("delete_untagged"),
DryRun: task.GetBoolFromExtraAttrs("dry_run"),
JobID: task.JobID,
CreationTime: task.CreationTime,
StartTime: task.StartTime,
UpdateTime: task.UpdateTime,
EndTime: task.EndTime,
}
}

56
src/controller/gc/controller_test.go
View File

@ -15,14 +15,14 @@ import (
type gcCtrTestSuite struct {
suite.Suite
scheduler *schedulertesting.Scheduler
execMgr *tasktesting.FakeExecutionManager
taskMgr *tasktesting.FakeManager
execMgr *tasktesting.ExecutionManager
taskMgr *tasktesting.Manager
ctl *controller
}
func (g *gcCtrTestSuite) SetupTest() {
g.execMgr = &tasktesting.FakeExecutionManager{}
g.taskMgr = &tasktesting.FakeManager{}
g.execMgr = &tasktesting.ExecutionManager{}
g.taskMgr = &tasktesting.Manager{}
g.scheduler = &schedulertesting.Scheduler{}
g.ctl = &controller{
taskMgr: g.taskMgr,
@ -37,15 +37,21 @@ func (g *gcCtrTestSuite) TestStart() {
g.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
dataMap := make(map[string]interface{})
g.Nil(g.ctl.Start(nil, dataMap))
p := Policy{
DeleteUntagged: true,
ExtraAttrs: dataMap,
}
id, err := g.ctl.Start(nil, p, task.ExecutionTriggerManual)
g.Nil(err)
g.Equal(int64(1), id)
}
func (g *gcCtrTestSuite) TestStop() {
g.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
g.execMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
g.Nil(g.ctl.Stop(nil, 1))
}
func (g *gcCtrTestSuite) TestLog() {
func (g *gcCtrTestSuite) TestGetTaskLog() {
g.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
{
ID: 1,
@ -55,19 +61,35 @@ func (g *gcCtrTestSuite) TestLog() {
}, nil)
g.taskMgr.On("GetLog", mock.Anything, mock.Anything).Return([]byte("hello world"), nil)
log, err := g.ctl.GetLog(nil, 1)
log, err := g.ctl.GetTaskLog(nil, 1)
g.Nil(err)
g.Equal([]byte("hello world"), log)
}
func (g *gcCtrTestSuite) TestCount() {
func (g *gcCtrTestSuite) TestExecutionCount() {
g.execMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil)
count, err := g.ctl.Count(nil, q.New(q.KeyWords{"VendorType": "gc"}))
count, err := g.ctl.ExecutionCount(nil, q.New(q.KeyWords{"VendorType": "gc"}))
g.Nil(err)
g.Equal(int64(1), count)
}
func (g *gcCtrTestSuite) TestHistory() {
func (g *gcCtrTestSuite) TestGetExecution() {
g.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
{
ID: 1,
Trigger: "Manual",
VendorType: GCVendorType,
StatusMessage: "Success",
},
}, nil)
hs, err := g.ctl.GetExecution(nil, int64(1))
g.Nil(err)
g.Equal("Manual", hs.Trigger)
}
func (g *gcCtrTestSuite) TestListExecutions() {
g.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
{
ID: 1,
@ -83,10 +105,10 @@ func (g *gcCtrTestSuite) TestHistory() {
},
}, nil)
hs, err := g.ctl.History(nil, q.New(q.KeyWords{"VendorType": "gc"}))
hs, err := g.ctl.ListExecutions(nil, q.New(q.KeyWords{"VendorType": "gc"}))
g.Nil(err)
g.Equal("Manual", hs[0].Kind)
g.Equal("Manual", hs[0].Trigger)
}
func (g *gcCtrTestSuite) TestGetSchedule() {
@ -103,11 +125,15 @@ func (g *gcCtrTestSuite) TestGetSchedule() {
}
func (g *gcCtrTestSuite) TestCreateSchedule() {
g.scheduler.On("Schedule", mock.Anything, mock.Anything, mock.Anything,
g.scheduler.On("Schedule", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
dataMap := make(map[string]interface{})
id, err := g.ctl.CreateSchedule(nil, "", dataMap)
p := Policy{
DeleteUntagged: true,
ExtraAttrs: dataMap,
}
id, err := g.ctl.CreateSchedule(nil, "Daily", "* * * * * *", p)
g.Nil(err)
g.Equal(int64(1), id)
}

62
src/controller/gc/model.go
View File

@ -4,29 +4,51 @@ import (
"time"
)
// Schedule ...
type Schedule struct {
Schedule *ScheduleParam `json:"schedule"`
// Policy ...
type Policy struct {
Trigger *Trigger `json:"trigger"`
DeleteUntagged bool `json:"bool"`
DryRun bool `json:"dryrun"`
ExtraAttrs map[string]interface{} `json:"extra_attrs"`
}
// ScheduleParam defines the parameter of schedule trigger
type ScheduleParam struct {
// Daily, Weekly, Custom, Manual, None
Type string `json:"type"`
// The cron string of scheduled job
// TriggerType represents the type of trigger.
type TriggerType string
// Trigger holds info for a trigger
type Trigger struct {
Type TriggerType `json:"type"`
Settings *TriggerSettings `json:"trigger_settings"`
}
// TriggerSettings is the setting about the trigger
type TriggerSettings struct {
Cron string `json:"cron"`
}
// History gc execution history
type History struct {
Schedule
ID int64 `json:"id"`
Name string `json:"job_name"`
Kind string `json:"job_kind"`
Parameters string `json:"job_parameters"`
Status string `json:"job_status"`
UUID string `json:"-"`
Deleted bool `json:"deleted"`
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
// Execution model for replication
type Execution struct {
ID int64
Status string
StatusMessage string
Trigger string
ExtraAttrs map[string]interface{}
StartTime time.Time
EndTime time.Time
}
// Task model for replication
type Task struct {
ID int64
ExecutionID int64
Status string
StatusMessage string
RunCount int32
DeleteUntagged bool
DryRun bool
JobID string
CreationTime time.Time
StartTime time.Time
UpdateTime time.Time
EndTime time.Time
}

16
src/pkg/task/model.go
View File

@ -115,6 +115,22 @@ func (t *Task) GetStringFromExtraAttrs(key string) string {
return str
}
// GetBoolFromExtraAttrs returns the bool value specified by key
func (t *Task) GetBoolFromExtraAttrs(key string) bool {
if len(t.ExtraAttrs) == 0 {
return false
}
rt, exist := t.ExtraAttrs[key]
if !exist {
return false
}
b, ok := rt.(bool)
if !ok {
return false
}
return b
}
// Job is the model represents the requested jobservice job
type Job struct {
Name string

130
src/server/v2.0/handler/gc.go
View File

@ -2,15 +2,19 @@ package handler
import (
"context"
"encoding/json"
"fmt"
"github.com/go-openapi/runtime/middleware"
"github.com/goharbor/harbor/src/controller/gc"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
"github.com/goharbor/harbor/src/server/v2.0/handler/model"
"github.com/goharbor/harbor/src/server/v2.0/models"
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/gc"
"os"
"strings"
)
type gcAPI struct {
@ -24,66 +28,93 @@ func newGCAPI() *gcAPI {
}
}
func (g *gcAPI) PostSchedule(ctx context.Context, params operation.PostScheduleParams) middleware.Responder {
if err := g.parseParam(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters); err != nil {
func (g *gcAPI) CreateGCSchedule(ctx context.Context, params operation.CreateGCScheduleParams) middleware.Responder {
id, err := g.kick(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters)
if err != nil {
return g.SendError(ctx, err)
}
return operation.NewPostScheduleOK()
// replace the /api/v2.0/system/gc/schedule/{id} to /api/v2.0/system/gc/{id}
lastSlashIndex := strings.LastIndex(params.HTTPRequest.URL.Path, "/")
if lastSlashIndex != -1 {
location := fmt.Sprintf("%s/%d", params.HTTPRequest.URL.Path[:lastSlashIndex], id)
return operation.NewCreateGCScheduleCreated().WithLocation(location)
}
return operation.NewCreateGCScheduleCreated()
}
func (g *gcAPI) PutSchedule(ctx context.Context, params operation.PutScheduleParams) middleware.Responder {
if err := g.parseParam(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters); err != nil {
func (g *gcAPI) UpdateGCSchedule(ctx context.Context, params operation.UpdateGCScheduleParams) middleware.Responder {
_, err := g.kick(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters)
if err != nil {
return g.SendError(ctx, err)
}
return operation.NewPutScheduleOK()
return operation.NewUpdateGCScheduleOK()
}
func (g *gcAPI) parseParam(ctx context.Context, scheType string, cron string, parameters map[string]interface{}) error {
func (g *gcAPI) kick(ctx context.Context, scheType string, cron string, parameters map[string]interface{}) (int64, error) {
// set the required parameters for GC
parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
parameters["time_window"] = config.GetGCTimeWindow()
var err error
var id int64
switch scheType {
case model.ScheduleManual:
err = g.gcCtr.Start(ctx, parameters)
case model.ScheduleNone:
case ScheduleManual:
policy := gc.Policy{
ExtraAttrs: parameters,
}
if dryRun, ok := parameters["dry_run"].(bool); ok {
policy.DryRun = dryRun
}
if deleteUntagged, ok := parameters["delete_untagged"].(bool); ok {
policy.DeleteUntagged = deleteUntagged
}
id, err = g.gcCtr.Start(ctx, policy, task.ExecutionTriggerManual)
case ScheduleNone:
err = g.gcCtr.DeleteSchedule(ctx)
case model.ScheduleHourly, model.ScheduleDaily, model.ScheduleWeekly, model.ScheduleCustom:
err = g.updateSchedule(ctx, cron, parameters)
case ScheduleHourly, ScheduleDaily, ScheduleWeekly, ScheduleCustom:
policy := gc.Policy{
ExtraAttrs: parameters,
}
if dryRun, ok := parameters["dry_run"].(bool); ok {
policy.DryRun = dryRun
}
if deleteUntagged, ok := parameters["delete_untagged"].(bool); ok {
policy.DeleteUntagged = deleteUntagged
}
err = g.updateSchedule(ctx, scheType, cron, policy)
}
return err
return id, err
}
func (g *gcAPI) createSchedule(ctx context.Context, cron string, parameters map[string]interface{}) error {
func (g *gcAPI) createSchedule(ctx context.Context, cronType, cron string, policy gc.Policy) error {
if cron == "" {
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("empty cron string for gc schedule")
}
_, err := g.gcCtr.CreateSchedule(ctx, cron, parameters)
_, err := g.gcCtr.CreateSchedule(ctx, cronType, cron, policy)
if err != nil {
return err
}
return nil
}
func (g *gcAPI) updateSchedule(ctx context.Context, cron string, parameters map[string]interface{}) error {
func (g *gcAPI) updateSchedule(ctx context.Context, cronType, cron string, policy gc.Policy) error {
if err := g.gcCtr.DeleteSchedule(ctx); err != nil {
return err
}
return g.createSchedule(ctx, cron, parameters)
return g.createSchedule(ctx, cronType, cron, policy)
}
func (g *gcAPI) GetSchedule(ctx context.Context, params operation.GetScheduleParams) middleware.Responder {
func (g *gcAPI) GetGCSchedule(ctx context.Context, params operation.GetGCScheduleParams) middleware.Responder {
schedule, err := g.gcCtr.GetSchedule(ctx)
if errors.IsNotFoundErr(err) {
return operation.NewGetScheduleOK().WithPayload(model.NewSchedule(&scheduler.Schedule{}).ToSwagger())
return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(&scheduler.Schedule{}).ToSwagger())
}
if err != nil {
return g.SendError(ctx, err)
}
return operation.NewGetScheduleOK().WithPayload(model.NewSchedule(schedule).ToSwagger())
return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(schedule).ToSwagger())
}
func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryParams) middleware.Responder {
@ -91,28 +122,75 @@ func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryP
if err != nil {
return g.SendError(ctx, err)
}
total, err := g.gcCtr.Count(ctx, query)
total, err := g.gcCtr.ExecutionCount(ctx, query)
if err != nil {
return g.SendError(ctx, err)
}
hs, err := g.gcCtr.History(ctx, query)
execs, err := g.gcCtr.ListExecutions(ctx, query)
if err != nil {
return g.SendError(ctx, err)
}
var hs []*model.GCHistory
for _, exec := range execs {
extraAttrsString, err := json.Marshal(exec.ExtraAttrs)
if err != nil {
return g.SendError(ctx, err)
}
hs = append(hs, &model.GCHistory{
ID: exec.ID,
Name: gc.GCVendorType,
Kind: exec.Trigger,
Parameters: string(extraAttrsString),
Schedule: &model.ScheduleParam{
Type: exec.Trigger,
},
Status: exec.Status,
CreationTime: exec.StartTime,
UpdateTime: exec.EndTime,
})
}
var results []*models.GCHistory
for _, h := range hs {
res := &model.GCHistory{}
res.History = h
results = append(results, res.ToSwagger())
results = append(results, h.ToSwagger())
}
return operation.NewGetGCHistoryOK().
WithXTotalCount(total).
WithLink(g.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
WithPayload(results)
}
func (g *gcAPI) GetGC(ctx context.Context, params operation.GetGCParams) middleware.Responder {
exec, err := g.gcCtr.GetExecution(ctx, params.GcID)
if err != nil {
return g.SendError(ctx, err)
}
extraAttrsString, err := json.Marshal(exec.ExtraAttrs)
if err != nil {
return g.SendError(ctx, err)
}
res := &model.GCHistory{
ID: exec.ID,
Name: gc.GCVendorType,
Kind: exec.Trigger,
Parameters: string(extraAttrsString),
Status: exec.Status,
Schedule: &model.ScheduleParam{
Type: exec.Trigger,
},
CreationTime: exec.StartTime,
UpdateTime: exec.EndTime,
}
return operation.NewGetGCOK().WithPayload(res.ToSwagger())
}
func (g *gcAPI) GetGCLog(ctx context.Context, params operation.GetGCLogParams) middleware.Responder {
log, err := g.gcCtr.GetLog(ctx, params.GcID)
log, err := g.gcCtr.GetTaskLog(ctx, params.GcID)
if err != nil {
return g.SendError(ctx, err)
}

40
src/server/v2.0/handler/model/gc.go
View File

@ -2,29 +2,31 @@ package model
import (
"github.com/go-openapi/strfmt"
"github.com/goharbor/harbor/src/controller/gc"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/server/v2.0/models"
"time"
)
const (
// ScheduleHourly : 'Hourly'
ScheduleHourly = "Hourly"
// ScheduleDaily : 'Daily'
ScheduleDaily = "Daily"
// ScheduleWeekly : 'Weekly'
ScheduleWeekly = "Weekly"
// ScheduleCustom : 'Custom'
ScheduleCustom = "Custom"
// ScheduleManual : 'Manual'
ScheduleManual = "Manual"
// ScheduleNone : 'None'
ScheduleNone = "None"
)
// ScheduleParam defines the parameter of schedule trigger
type ScheduleParam struct {
// Daily, Weekly, Custom, Manual, None
Type string `json:"type"`
// The cron string of scheduled job
Cron string `json:"cron"`
}
// GCHistory ...
// GCHistory gc execution history
type GCHistory struct {
*gc.History
Schedule *ScheduleParam `json:"schedule"`
ID int64 `json:"id"`
Name string `json:"job_name"`
Kind string `json:"job_kind"`
Parameters string `json:"job_parameters"`
Status string `json:"job_status"`
UUID string `json:"-"`
Deleted bool `json:"deleted"`
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
}
// ToSwagger converts the history to the swagger model
@ -37,8 +39,8 @@ func (h *GCHistory) ToSwagger() *models.GCHistory {
Deleted: h.Deleted,
JobStatus: h.Status,
Schedule: &models.ScheduleObj{
Cron: h.Schedule.Schedule.Cron,
Type: h.Schedule.Schedule.Type,
Cron: h.Schedule.Cron,
Type: h.Schedule.Type,
},
CreationTime: strfmt.DateTime(h.CreationTime),
UpdateTime: strfmt.DateTime(h.UpdateTime),

15
src/server/v2.0/handler/util.go
View File

@ -29,6 +29,21 @@ import (
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
)
const (
// ScheduleHourly : 'Hourly'
ScheduleHourly = "Hourly"
// ScheduleDaily : 'Daily'
ScheduleDaily = "Daily"
// ScheduleWeekly : 'Weekly'
ScheduleWeekly = "Weekly"
// ScheduleCustom : 'Custom'
ScheduleCustom = "Custom"
// ScheduleManual : 'Manual'
ScheduleManual = "Manual"
// ScheduleNone : 'None'
ScheduleNone = "None"
)
func boolValue(v *bool) bool {
if v != nil {
return *v

1
tests/apitests/python/library/base.py
View File

@ -61,6 +61,7 @@ def _create_client(server, credential, debug, api_type="products"):
"scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)),
"replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)),
"robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)),
"gc": v2_swagger_client.GcApi(v2_swagger_client.ApiClient(cfg)),
}.get(api_type,'Error: Wrong API type')
def _assert_status_code(expect_code, return_code):

134
tests/apitests/python/library/gc.py Normal file
View File

@ -0,0 +1,134 @@
# -*- coding: utf-8 -*-
import time
import base
import re
import v2_swagger_client
from v2_swagger_client.rest import ApiException
class GC(base.Base, object):
def __init__(self):
super(GC,self).__init__(api_type = "gc")
def get_gc_history(self, expect_status_code = 200, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
try:
data, status_code, _ = client.get_gc_history_with_http_info()
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return data
def get_gc_status_by_id(self, job_id, expect_status_code = 200, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
try:
data, status_code, _ = client.get_gc_with_http_info(job_id)
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return data
def get_gc_log_by_id(self, job_id, expect_status_code = 200, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
try:
data, status_code, _ = client.get_gc_log_with_http_info(job_id)
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return data
def get_gc_schedule(self, expect_status_code = 200, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
try:
data, status_code, _ = client.get_gc_schedule_with_http_info()
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return data
def create_gc_schedule(self, schedule_type, is_delete_untagged, cron = None, expect_status_code = 201, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
gc_parameters = {'delete_untagged':is_delete_untagged}
gc_schedule = v2_swagger_client.ScheduleObj()
gc_schedule.type = schedule_type
if cron is not None:
gc_schedule.cron = cron
gc_job = v2_swagger_client.Schedule()
gc_job.schedule = gc_schedule
gc_job.parameters = gc_parameters
try:
_, status_code, header = client.create_gc_schedule_with_http_info(gc_job)
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Create GC schedule response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Create GC schedule result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return base._get_id_from_header(header)
def gc_now(self, is_delete_untagged=False, **kwargs):
gc_id = self.create_gc_schedule('Manual', is_delete_untagged, **kwargs)
return gc_id
def validate_gc_job_status(self, gc_id, expected_gc_status, **kwargs):
get_gc_status_finish = False
timeout_count = 20
while timeout_count > 0:
time.sleep(5)
status = self.get_gc_status_by_id(gc_id, **kwargs)
print("GC job No: {}, status: {}".format(timeout_count, status.job_status))
if status.job_status == expected_gc_status:
get_gc_status_finish = True
break
timeout_count = timeout_count - 1
if not (get_gc_status_finish):
raise Exception("GC status is not as expected '{}' actual GC status is '{}'".format(expected_gc_status, status.job_status))
def validate_deletion_success(self, gc_id, **kwargs):
log_content = self.get_gc_log_by_id(gc_id, **kwargs)
key_message = "manifests eligible for deletion"
key_message_pos = log_content.find(key_message)
full_message = log_content[key_message_pos-30 : key_message_pos + len(key_message)]
deleted_files_count_list = re.findall(r'\s+(\d+)\s+blobs\s+and\s+\d+\s+manifests\s+eligible\s+for\s+deletion', full_message)
if len(deleted_files_count_list) != 1:
raise Exception(r"Fail to get blobs eligible for deletion in log file, failure is {}.".format(len(deleted_files_count_list)))
deleted_files_count = int(deleted_files_count_list[0])
if deleted_files_count == 0:
raise Exception(r"Get blobs eligible for deletion count is {}, while we expect more than 1.".format(deleted_files_count))

32
tests/apitests/python/library/system.py
View File

@ -157,38 +157,6 @@ class System(base.Base):
scan_all_id = self.create_scan_all_schedule('Manual', **kwargs)
return scan_all_id
def gc_now(self, is_delete_untagged=False, **kwargs):
gc_id = self.create_gc_schedule('Manual', is_delete_untagged, **kwargs)
return gc_id
def validate_gc_job_status(self, gc_id, expected_gc_status, **kwargs):
get_gc_status_finish = False
timeout_count = 20
while timeout_count > 0:
time.sleep(5)
status = self.get_gc_status_by_id(gc_id, **kwargs)
print("GC job No: {}, status: {}".format(timeout_count, status.job_status))
if status.job_status == expected_gc_status:
get_gc_status_finish = True
break
timeout_count = timeout_count - 1
if not (get_gc_status_finish):
raise Exception("GC status is not as expected '{}' actual GC status is '{}'".format(expected_gc_status, status.job_status))
def validate_deletion_success(self, gc_id, **kwargs):
log_content = self.get_gc_log_by_id(gc_id, **kwargs)
key_message = "manifests eligible for deletion"
key_message_pos = log_content.find(key_message)
full_message = log_content[key_message_pos-30 : key_message_pos + len(key_message)]
deleted_files_count_list = re.findall(r'\s+(\d+)\s+blobs\s+and\s+\d+\s+manifests\s+eligible\s+for\s+deletion', full_message)
if len(deleted_files_count_list) != 1:
raise Exception(r"Fail to get blobs eligible for deletion in log file, failure is {}.".format(len(deleted_files_count_list)))
deleted_files_count = int(deleted_files_count_list[0])
if deleted_files_count == 0:
raise Exception(r"Get blobs eligible for deletion count is {}, while we expect more than 1.".format(deleted_files_count))
def set_cve_allowlist(self, expires_at=None, expected_status_code=200, *cve_ids, **kwargs):
client = self._get_client(**kwargs)
cve_list = [swagger_client.CVEAllowlistItem(cve_id=c) for c in cve_ids]

16
tests/apitests/python/test_garbage_collection.py
View File

@ -7,17 +7,17 @@ from testutils import ADMIN_CLIENT, suppress_urllib3_warning
from testutils import TEARDOWN
from testutils import harbor_server
from library.user import User
from library.system import System
from library.project import Project
from library.repository import Repository
from library.base import _assert_status_code
from library.repository import push_special_image_to_project
from library.artifact import Artifact
from library.gc import GC
class TestProjects(unittest.TestCase):
@suppress_urllib3_warning
def setUp(self):
self.system = System()
self.gc = GC()
self.project = Project()
self.user = User()
self.repo = Repository()
@ -82,13 +82,13 @@ class TestProjects(unittest.TestCase):
self.artifact.delete_tag(TestProjects.project_gc_untag_name, self.repo_name_untag, self.tag, self.tag, **ADMIN_CLIENT)
#5. Tigger garbage collection operation;
gc_id = self.system.gc_now(**ADMIN_CLIENT)
gc_id = self.gc.gc_now(**ADMIN_CLIENT)
#6. Check garbage collection job was finished;
self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT)
self.gc.validate_gc_job_status(gc_id, "Success", **ADMIN_CLIENT)
#7. Get garbage collection log, check there is a number of files was deleted;
self.system.validate_deletion_success(gc_id, **ADMIN_CLIENT)
self.gc.validate_deletion_success(gc_id, **ADMIN_CLIENT)
artifacts = self.artifact.list_artifacts(TestProjects.project_gc_untag_name, self.repo_name_untag, **TestProjects.USER_GC_CLIENT)
_assert_status_code(len(artifacts), 1)
@ -96,13 +96,13 @@ class TestProjects(unittest.TestCase):
time.sleep(5)
#9. Tigger garbage collection operation;
gc_id = self.system.gc_now(is_delete_untagged=True, **ADMIN_CLIENT)
gc_id = self.gc.gc_now(is_delete_untagged=True, **ADMIN_CLIENT)
#10. Check garbage collection job was finished;
self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT)
self.gc.validate_gc_job_status(gc_id, "Success", **ADMIN_CLIENT)
#7. Get garbage collection log, check there is a number of files was deleted;
self.system.validate_deletion_success(gc_id, **ADMIN_CLIENT)
self.gc.validate_deletion_success(gc_id, **ADMIN_CLIENT)
#11. Repository with untag image should be still there;
repo_data_untag = self.repo.list_repositories(TestProjects.project_gc_untag_name, **TestProjects.USER_GC_CLIENT)