Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #13139 from wy65701436/migrate-gc 

Migrate gc to task manager
This commit is contained in:
Wenkai Yin(尹文开) 2020-12-14 10:43:44 +08:00 committed by GitHub
parent 0a38ae1843 39e1a4f2b4
commit 6569016d35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 1179 additions and 414 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
api/v2.0/legacy_swagger.yaml
View File

@ -1549,25 +1549,6 @@ paths:
description: Only admin has this authority.
'500':
description: Unexpected internal errors.
/system/gc:
get:
summary: Get gc results.
description: This endpoint let user get latest ten gc results.
tags:
- Products
responses:
'200':
description: Get gc results successfully.
schema:
type: array
items:
$ref: '#/definitions/GCResult'
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'500':
description: Unexpected internal errors.
'/system/gc/{id}':
get:
summary: Get gc status.
@ -1592,101 +1573,6 @@ paths:
description: User does not have permission of admin role.
'500':
description: Unexpected internal errors.
'/system/gc/{id}/log':
get:
summary: Get gc job log.
description: This endpoint let user get gc job logs filtered by specific ID.
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Relevant job ID
tags:
- Products
responses:
'200':
description: Get successfully.
schema:
type: string
'400':
description: Illegal format of provided ID value.
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'404':
description: The specific gc ID's log does not exist.
'500':
description: Unexpected internal errors.
/system/gc/schedule:
get:
summary: Get gc's schedule.
description: This endpoint is for get schedule of gc job.
tags:
- Products
responses:
'200':
description: Get gc's schedule.
schema:
$ref: '#/definitions/AdminJobSchedule'
'401':
description: User need to log in first.
'403':
description: Only admin has this authority.
'500':
description: Unexpected internal errors.
put:
summary: Update gc's schedule.
description: |
This endpoint is for update gc schedule.
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/AdminJobSchedule'
description: Updates of gc's schedule.
tags:
- Products
responses:
'200':
description: Updated gc's schedule successfully.
'400':
description: Invalid schedule type.
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'500':
description: Unexpected internal errors.
post:
summary: Create a gc schedule.
description: |
This endpoint is for update gc schedule.
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/AdminJobSchedule'
description: Updates of gc's schedule.
tags:
- Products
responses:
'200':
description: GC schedule successfully.
'400':
description: Invalid schedule type.
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'409':
description: There is a "gc" job in progress, so the request cannot be served.
'500':
description: Unexpected internal errors.
/system/scanAll/schedule:
get:
summary: Get scan_all's schedule.

208
api/v2.0/swagger.yaml
View File

@ -2017,6 +2017,155 @@ paths:
description: Not found the default root certificate.
'500':
$ref: '#/responses/500'
/system/gc:
get:
summary: Get gc results.
description: This endpoint let user get gc execution history.
tags:
- gc
operationId: getGCHistory
parameters:
- $ref: '#/parameters/query'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
responses:
'200':
description: Get gc results successfully.
headers:
X-Total-Count:
description: The total count of history
type: integer
Link:
description: Link refers to the previous page and next page
type: string
schema:
type: array
items:
$ref: '#/definitions/GCHistory'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
/system/gc/{gc_id}:
get:
summary: Get gc status.
description: This endpoint let user get gc status filtered by specific ID.
operationId: getGC
parameters:
- $ref: '#/parameters/gcId'
tags:
- gc
responses:
'200':
description: Get gc results successfully.
schema:
$ref: '#/definitions/GCHistory'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
/system/gc/{gc_id}/log:
get:
summary: Get gc job log.
description: This endpoint let user get gc job logs filtered by specific ID.
operationId: getGCLog
parameters:
- $ref: '#/parameters/gcId'
tags:
- gc
produces:
- text/plain
responses:
'200':
description: Get successfully.
schema:
type: string
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
/system/gc/schedule:
get:
summary: Get gc's schedule.
description: This endpoint is for get schedule of gc job.
operationId: getGCSchedule
tags:
- gc
responses:
'200':
description: Get gc's schedule.
schema:
$ref: '#/definitions/GCHistory'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
post:
summary: Create a gc schedule.
description: |
This endpoint is for update gc schedule.
operationId: createGCSchedule
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/Schedule'
description: Updates of gc's schedule.
tags:
- gc
responses:
'201':
$ref: '#/responses/201'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'409':
$ref: '#/responses/409'
'500':
$ref: '#/responses/500'
put:
summary: Update gc's schedule.
description: |
This endpoint is for update gc schedule.
operationId: updateGCSchedule
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/Schedule'
description: Updates of gc's schedule.
tags:
- gc
responses:
'200':
description: Updated gc's schedule successfully.
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
/ping:
get:
summary: Ping Harbor to check if it's alive.
@ -2134,6 +2283,13 @@ parameters:
description: Robot ID
required: true
type: integer
gcId:
name: gc_id
in: path
description: The ID of the gc log
required: true
type: integer
format: int64
responses:
'200':
description: Success
@ -3351,3 +3507,55 @@ definitions:
description: The storage of system.
items:
$ref: '#/definitions/Storage'
GCHistory:
type: object
properties:
id:
type: integer
description: the id of gc job.
job_name:
type: string
description: the job name of gc job.
job_kind:
type: string
description: the job kind of gc job.
job_parameters:
type: string
description: the job parameters of gc job.
schedule:
$ref: '#/definitions/ScheduleObj'
job_status:
type: string
description: the status of gc job.
deleted:
type: boolean
description: if gc job was deleted.
creation_time:
type: string
format: date-time
description: the creation time of gc job.
update_time:
type: string
format: date-time
description: the update time of gc job.
Schedule:
type: object
properties:
schedule:
$ref: '#/definitions/ScheduleObj'
parameters:
type: object
description: The parameters of admin job
additionalProperties:
type: object
ScheduleObj:
type: object
properties:
type:
type: string
description: |
The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manually' and 'None'.
'Manually' means to trigger it right away and 'None' means to cancel the schedule.
cron:
type: string
description: A cron expression, a time-based job scheduler.

6
make/migrations/postgresql/0050_2.2.0_schema.up.sql
View File

@ -1,4 +1,4 @@
/*
/*
Fixes issue https://github.com/goharbor/harbor/issues/13317
Ensure the role_id of maintainer is 4 and the role_id of limisted guest is 5
*/
@ -268,3 +268,7 @@ BEGIN
UPDATE scanner_registration SET is_default = TRUE WHERE name = 'Trivy' AND immutable = TRUE;
END IF;
END $$;
ALTER TABLE execution ALTER COLUMN vendor_type type varchar(64);
ALTER TABLE schedule ALTER COLUMN vendor_type type varchar(64);
ALTER TABLE schedule ADD COLUMN IF NOT EXISTS extra_attrs JSON;
ALTER TABLE task ALTER COLUMN vendor_type type varchar(64);

27
src/controller/gc/callback.go Normal file
View File

@ -0,0 +1,27 @@
package gc
import (
"context"
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
)
func init() {
err := scheduler.RegisterCallbackFunc(SchedulerCallback, gcCallback)
if err != nil {
log.Fatalf("failed to registry GC call back, %v", err)
}
}
func gcCallback(ctx context.Context, p string) error {
param := &Policy{}
if err := json.Unmarshal([]byte(p), param); err != nil {
return fmt.Errorf("failed to unmarshal the param: %v", err)
}
_, err := Ctl.Start(orm.Context(), *param, task.ExecutionTriggerSchedule)
return err
}

217
src/controller/gc/controller.go Normal file
View File

@ -0,0 +1,217 @@
package gc
import (
"context"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
)
var (
// Ctl is a global garbage collection controller instance
Ctl = NewController()
)
const (
// SchedulerCallback ...
SchedulerCallback = "GARBAGE_COLLECTION"
// GCVendorType ...
GCVendorType = "GARBAGE_COLLECTION"
)
// Controller manages the tags
type Controller interface {
// Start start a manual gc job
Start(ctx context.Context, policy Policy, trigger string) (int64, error)
// Stop stop a gc job
Stop(ctx context.Context, id int64) error
// ExecutionCount returns the total count of executions according to the query
ExecutionCount(ctx context.Context, query *q.Query) (count int64, err error)
// ListExecutions lists the executions according to the query
ListExecutions(ctx context.Context, query *q.Query) (executions []*Execution, err error)
// GetExecution gets the specific execution
GetExecution(ctx context.Context, executionID int64) (execution *Execution, err error)
// GetTask gets the specific task
GetTask(ctx context.Context, id int64) (*Task, error)
// GetTaskLog gets log of the specific task
GetTaskLog(ctx context.Context, id int64) ([]byte, error)
// GetSchedule get the current gc schedule
GetSchedule(ctx context.Context) (*scheduler.Schedule, error)
// CreateSchedule create the gc schedule with cron type & string
CreateSchedule(ctx context.Context, cronType, cron string, policy Policy) (int64, error)
// DeleteSchedule remove the gc schedule
DeleteSchedule(ctx context.Context) error
}
// NewController creates an instance of the default repository controller
func NewController() Controller {
return &controller{
taskMgr: task.NewManager(),
exeMgr: task.NewExecutionManager(),
schedulerMgr: scheduler.New(),
}
}
type controller struct {
taskMgr task.Manager
exeMgr task.ExecutionManager
schedulerMgr scheduler.Scheduler
}
// Start starts the manual GC
func (c *controller) Start(ctx context.Context, policy Policy, trigger string) (int64, error) {
para := make(map[string]interface{})
para["delete_untagged"] = policy.DeleteUntagged
para["dry_run"] = policy.DryRun
para["redis_url_reg"] = policy.ExtraAttrs["redis_url_reg"]
para["time_window"] = policy.ExtraAttrs["time_window"]
execID, err := c.exeMgr.Create(ctx, GCVendorType, -1, trigger, para)
if err != nil {
return -1, err
}
_, err = c.taskMgr.Create(ctx, execID, &task.Job{
Name: job.ImageGC,
Metadata: &job.Metadata{
JobKind: job.KindGeneric,
},
Parameters: para,
})
if err != nil {
return -1, err
}
return execID, nil
}
// Stop ...
func (c *controller) Stop(ctx context.Context, id int64) error {
return c.exeMgr.Stop(ctx, id)
}
// ExecutionCount ...
func (c *controller) ExecutionCount(ctx context.Context, query *q.Query) (int64, error) {
query.Keywords["VendorType"] = GCVendorType
return c.exeMgr.Count(ctx, query)
}
// ListExecutions ...
func (c *controller) ListExecutions(ctx context.Context, query *q.Query) ([]*Execution, error) {
query = q.MustClone(query)
query.Keywords["VendorType"] = GCVendorType
execs, err := c.exeMgr.List(ctx, query)
if err != nil {
return nil, err
}
var executions []*Execution
for _, exec := range execs {
executions = append(executions, convertExecution(exec))
}
return executions, nil
}
// GetExecution ...
func (c *controller) GetExecution(ctx context.Context, id int64) (*Execution, error) {
execs, err := c.exeMgr.List(ctx, &q.Query{
Keywords: map[string]interface{}{
"ID": id,
"VendorType": GCVendorType,
},
})
if err != nil {
return nil, err
}
if len(execs) == 0 {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).
WithMessage("garbage collection execution %d not found", id)
}
return convertExecution(execs[0]), nil
}
// GetTask ...
func (c *controller) GetTask(ctx context.Context, id int64) (*Task, error) {
tasks, err := c.taskMgr.List(ctx, &q.Query{
Keywords: map[string]interface{}{
"ID": id,
"VendorType": GCVendorType,
},
})
if err != nil {
return nil, err
}
if len(tasks) == 0 {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).
WithMessage("garbage collection task %d not found", id)
}
return convertTask(tasks[0]), nil
}
// GetTaskLog ...
func (c *controller) GetTaskLog(ctx context.Context, id int64) ([]byte, error) {
_, err := c.GetTask(ctx, id)
if err != nil {
return nil, err
}
return c.taskMgr.GetLog(ctx, id)
}
// GetSchedule ...
func (c *controller) GetSchedule(ctx context.Context) (*scheduler.Schedule, error) {
sch, err := c.schedulerMgr.ListSchedules(ctx, q.New(q.KeyWords{"VendorType": GCVendorType}))
if err != nil {
return nil, err
}
if len(sch) == 0 {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc schedule is found")
}
if sch[0] == nil {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc schedule is found")
}
return sch[0], nil
}
// CreateSchedule ...
func (c *controller) CreateSchedule(ctx context.Context, cronType, cron string, policy Policy) (int64, error) {
extras := make(map[string]interface{})
extras["delete_untagged"] = policy.DeleteUntagged
return c.schedulerMgr.Schedule(ctx, GCVendorType, -1, cronType, cron, SchedulerCallback, policy, extras)
}
// DeleteSchedule ...
func (c *controller) DeleteSchedule(ctx context.Context) error {
return c.schedulerMgr.UnScheduleByVendor(ctx, GCVendorType, -1)
}
func convertExecution(exec *task.Execution) *Execution {
return &Execution{
ID: exec.ID,
Status: exec.Status,
StatusMessage: exec.StatusMessage,
Trigger: exec.Trigger,
ExtraAttrs: exec.ExtraAttrs,
StartTime: exec.StartTime,
EndTime: exec.EndTime,
}
}
func convertTask(task *task.Task) *Task {
return &Task{
ID: task.ID,
ExecutionID: task.ExecutionID,
Status: task.Status,
StatusMessage: task.StatusMessage,
RunCount: task.RunCount,
DeleteUntagged: task.GetBoolFromExtraAttrs("delete_untagged"),
DryRun: task.GetBoolFromExtraAttrs("dry_run"),
JobID: task.JobID,
CreationTime: task.CreationTime,
StartTime: task.StartTime,
UpdateTime: task.UpdateTime,
EndTime: task.EndTime,
}
}

148
src/controller/gc/controller_test.go Normal file
View File

@ -0,0 +1,148 @@
package gc
import (
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
"github.com/goharbor/harbor/src/testing/mock"
schedulertesting "github.com/goharbor/harbor/src/testing/pkg/scheduler"
tasktesting "github.com/goharbor/harbor/src/testing/pkg/task"
"github.com/stretchr/testify/suite"
"testing"
)
type gcCtrTestSuite struct {
suite.Suite
scheduler *schedulertesting.Scheduler
execMgr *tasktesting.ExecutionManager
taskMgr *tasktesting.Manager
ctl *controller
}
func (g *gcCtrTestSuite) SetupTest() {
g.execMgr = &tasktesting.ExecutionManager{}
g.taskMgr = &tasktesting.Manager{}
g.scheduler = &schedulertesting.Scheduler{}
g.ctl = &controller{
taskMgr: g.taskMgr,
exeMgr: g.execMgr,
schedulerMgr: g.scheduler,
}
}
func (g *gcCtrTestSuite) TestStart() {
g.execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
g.taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
g.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
dataMap := make(map[string]interface{})
p := Policy{
DeleteUntagged: true,
ExtraAttrs: dataMap,
}
id, err := g.ctl.Start(nil, p, task.ExecutionTriggerManual)
g.Nil(err)
g.Equal(int64(1), id)
}
func (g *gcCtrTestSuite) TestStop() {
g.execMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
g.Nil(g.ctl.Stop(nil, 1))
}
func (g *gcCtrTestSuite) TestGetTaskLog() {
g.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
{
ID: 1,
ExecutionID: 1,
Status: job.SuccessStatus.String(),
},
}, nil)
g.taskMgr.On("GetLog", mock.Anything, mock.Anything).Return([]byte("hello world"), nil)
log, err := g.ctl.GetTaskLog(nil, 1)
g.Nil(err)
g.Equal([]byte("hello world"), log)
}
func (g *gcCtrTestSuite) TestExecutionCount() {
g.execMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil)
count, err := g.ctl.ExecutionCount(nil, q.New(q.KeyWords{"VendorType": "gc"}))
g.Nil(err)
g.Equal(int64(1), count)
}
func (g *gcCtrTestSuite) TestGetExecution() {
g.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
{
ID: 1,
Trigger: "Manual",
VendorType: GCVendorType,
StatusMessage: "Success",
},
}, nil)
hs, err := g.ctl.GetExecution(nil, int64(1))
g.Nil(err)
g.Equal("Manual", hs.Trigger)
}
func (g *gcCtrTestSuite) TestListExecutions() {
g.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
{
ID: 1,
Trigger: "Manual",
},
}, nil)
g.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
{
ID: 112,
ExecutionID: 1,
Status: job.SuccessStatus.String(),
},
}, nil)
hs, err := g.ctl.ListExecutions(nil, q.New(q.KeyWords{"VendorType": "gc"}))
g.Nil(err)
g.Equal("Manual", hs[0].Trigger)
}
func (g *gcCtrTestSuite) TestGetSchedule() {
g.scheduler.On("ListSchedules", mock.Anything, mock.Anything).Return([]*scheduler.Schedule{
{
ID: 1,
VendorType: "gc",
},
}, nil)
sche, err := g.ctl.GetSchedule(nil)
g.Nil(err)
g.Equal("gc", sche.VendorType)
}
func (g *gcCtrTestSuite) TestCreateSchedule() {
g.scheduler.On("Schedule", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
dataMap := make(map[string]interface{})
p := Policy{
DeleteUntagged: true,
ExtraAttrs: dataMap,
}
id, err := g.ctl.CreateSchedule(nil, "Daily", "* * * * * *", p)
g.Nil(err)
g.Equal(int64(1), id)
}
func (g *gcCtrTestSuite) TestDeleteSchedule() {
g.scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything, mock.Anything).Return(nil)
g.Nil(g.ctl.DeleteSchedule(nil))
}
func TestControllerTestSuite(t *testing.T) {
suite.Run(t, &gcCtrTestSuite{})
}

54
src/controller/gc/model.go Normal file
View File

@ -0,0 +1,54 @@
package gc
import (
"time"
)
// Policy ...
type Policy struct {
Trigger *Trigger `json:"trigger"`
DeleteUntagged bool `json:"bool"`
DryRun bool `json:"dryrun"`
ExtraAttrs map[string]interface{} `json:"extra_attrs"`
}
// TriggerType represents the type of trigger.
type TriggerType string
// Trigger holds info for a trigger
type Trigger struct {
Type TriggerType `json:"type"`
Settings *TriggerSettings `json:"trigger_settings"`
}
// TriggerSettings is the setting about the trigger
type TriggerSettings struct {
Cron string `json:"cron"`
}
// Execution model for replication
type Execution struct {
ID int64
Status string
StatusMessage string
Trigger string
ExtraAttrs map[string]interface{}
StartTime time.Time
EndTime time.Time
}
// Task model for replication
type Task struct {
ID int64
ExecutionID int64
Status string
StatusMessage string
RunCount int32
DeleteUntagged bool
DryRun bool
JobID string
CreationTime time.Time
StartTime time.Time
UpdateTime time.Time
EndTime time.Time
}

6
src/controller/p2p/preheat/controller.go
View File

@ -292,8 +292,9 @@ func (c *controller) CreatePolicy(ctx context.Context, schema *policyModels.Sche
schema.Trigger.Type == policyModels.TriggerTypeScheduled &&
len(schema.Trigger.Settings.Cron) > 0 {
// schedule and update policy
extras := make(map[string]interface{})
if _, err = c.scheduler.Schedule(ctx, job.P2PPreheat, id, "", schema.Trigger.Settings.Cron,
SchedulerCallback, TriggerParam{PolicyID: id}); err != nil {
SchedulerCallback, TriggerParam{PolicyID: id}, extras); err != nil {
return 0, err
}
@ -384,8 +385,9 @@ func (c *controller) UpdatePolicy(ctx context.Context, schema *policyModels.Sche
// schedule new
if needSch {
extras := make(map[string]interface{})
if _, err := c.scheduler.Schedule(ctx, job.P2PPreheat, schema.ID, "", cron, SchedulerCallback,
TriggerParam{PolicyID: schema.ID}); err != nil {
TriggerParam{PolicyID: schema.ID}, extras); err != nil {
return err
}
}

2
src/controller/p2p/preheat/controllor_test.go
View File

@ -241,7 +241,7 @@ func (s *preheatSuite) TestCreatePolicy() {
FiltersStr: `[{"type":"repository","value":"harbor*"},{"type":"tag","value":"2*"}]`,
TriggerStr: fmt.Sprintf(`{"type":"%s", "trigger_setting":{"cron":"* * * * */1"}}`, policy.TriggerTypeScheduled),
}
s.fakeScheduler.On("Schedule", s.ctx, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
s.fakeScheduler.On("Schedule", s.ctx, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
s.fakePolicyMgr.On("Create", s.ctx, policy).Return(int64(1), nil)
s.fakePolicyMgr.On("Update", s.ctx, mock.Anything, mock.Anything).Return(nil)
s.fakeScheduler.On("UnScheduleByVendor", s.ctx, mock.Anything, mock.Anything).Return(nil)

33
src/core/api/harborapi_test.go
View File

@ -122,9 +122,6 @@ func init() {
beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping")
beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List")
beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete")
beego.Router("/api/system/gc/:id", &GCAPI{}, "get:GetGC")
beego.Router("/api/system/gc/:id([0-9]+)/log", &GCAPI{}, "get:GetLog")
beego.Router("/api/system/gc/schedule", &GCAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/system/scanAll/schedule", &ScanAllAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/system/CVEAllowlist", &SysCVEAllowlistAPI{}, "get:Get;put:Put")
beego.Router("/api/system/oidc/ping", &OIDCAPI{}, "post:Ping")
@ -864,36 +861,6 @@ func (a testapi) DeleteMeta(authInfor usrInfo, projectID int64, name string) (in
return code, string(body), err
}
func (a testapi) AddGC(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) {
_sling := sling.New().Post(a.basePath)
path := "/api/system/gc/schedule"
_sling = _sling.Path(path)
// body params
_sling = _sling.BodyJSON(adminReq)
var httpStatusCode int
var err error
httpStatusCode, _, err = request(_sling, jsonAcceptHeader, authInfor)
return httpStatusCode, err
}
func (a testapi) GCScheduleGet(authInfo usrInfo) (int, api_models.AdminJobSchedule, error) {
_sling := sling.New().Get(a.basePath)
path := "/api/system/gc/schedule"
_sling = _sling.Path(path)
httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo)
var successPayLoad api_models.AdminJobSchedule
if 200 == httpStatusCode && nil == err {
err = json.Unmarshal(body, &successPayLoad)
}
return httpStatusCode, successPayLoad, err
}
func (a testapi) AddScanAll(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) {
_sling := sling.New().Post(a.basePath)

147
src/core/api/reg_gc.go
View File

@ -1,147 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"errors"
"github.com/goharbor/harbor/src/core/config"
"net/http"
"os"
"strconv"
common_job "github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/core/api/models"
)
// GCAPI handles request of harbor GC...
type GCAPI struct {
AJAPI
}
// Prepare validates the URL and parms, it needs the system admin permission.
func (gc *GCAPI) Prepare() {
gc.BaseController.Prepare()
if !gc.SecurityCtx.IsAuthenticated() {
gc.SendUnAuthorizedError(errors.New("UnAuthorized"))
return
}
if !gc.SecurityCtx.IsSysAdmin() {
gc.SendForbiddenError(errors.New(gc.SecurityCtx.GetUsername()))
return
}
}
// Post according to the request, it creates a cron schedule or a manual trigger for GC.
// create a daily schedule for GC
// {
// "schedule": {
// "type": "Daily",
// "cron": "0 0 0 * * *"
// },
// "parameters": {
// "delete_untagged": true
// }
// }
// create a manual trigger for GC
// {
// "schedule": {
// "type": "Manual"
// },
// "parameters": {
// "delete_untagged": true
// "read_only": true
// }
// }
func (gc *GCAPI) Post() {
parameters := make(map[string]interface{})
ajr := models.AdminJobReq{
Parameters: parameters,
}
isValid, err := gc.DecodeJSONReqAndValidate(&ajr)
if !isValid {
gc.SendBadRequestError(err)
return
}
ajr.Parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
// default is the non-blocking GC job.
ajr.Name = common_job.ImageGC
ajr.Parameters["time_window"] = config.GetGCTimeWindow()
// if specify read_only:true, API will submit the readonly GC job, otherwise default is non-blocking GC job.
readOnlyParam, exist := ajr.Parameters["read_only"]
if exist {
if readOnly, ok := readOnlyParam.(bool); ok && readOnly {
ajr.Name = common_job.ImageGCReadOnly
}
}
gc.submit(&ajr)
gc.Redirect(http.StatusCreated, strconv.FormatInt(ajr.ID, 10))
}
// Put handles GC cron schedule update/delete.
// Request: delete the schedule of GC
// {
// "schedule": {
// "type": "None",
// "cron": ""
// },
// "parameters": {
// "delete_untagged": true
// }
// }
func (gc *GCAPI) Put() {
parameters := make(map[string]interface{})
ajr := models.AdminJobReq{
Parameters: parameters,
}
isValid, err := gc.DecodeJSONReqAndValidate(&ajr)
if !isValid {
gc.SendBadRequestError(err)
return
}
ajr.Name = common_job.ImageGC
ajr.Parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
ajr.Parameters["time_window"] = config.GetGCTimeWindow()
gc.updateSchedule(ajr)
}
// GetGC ...
func (gc *GCAPI) GetGC() {
id, err := gc.GetInt64FromPath(":id")
if err != nil {
gc.SendInternalServerError(errors.New("need to specify gc id"))
return
}
gc.get(id)
}
// List returns the top 10 executions of GC which includes manual and cron.
func (gc *GCAPI) List() {
gc.list(common_job.ImageGC)
}
// Get gets GC schedule ...
func (gc *GCAPI) Get() {
gc.getSchedule(common_job.ImageGC)
}
// GetLog ...
func (gc *GCAPI) GetLog() {
id, err := gc.GetInt64FromPath(":id")
if err != nil {
gc.SendBadRequestError(errors.New("invalid ID"))
return
}
gc.getLog(id)
}

39
src/core/api/reg_gc_test.go
View File

@ -1,39 +0,0 @@
package api
import (
"testing"
"github.com/goharbor/harbor/src/testing/apitests/apilib"
"github.com/stretchr/testify/assert"
)
func TestGCPost(t *testing.T) {
adminJob001 := apilib.AdminJobReq{
Parameters: map[string]interface{}{"delete_untagged": false},
}
assert := assert.New(t)
apiTest := newHarborAPI()
// case 1: add a new admin job
code, err := apiTest.AddGC(*admin, adminJob001)
if err != nil {
t.Error("Error occurred while add a admin job", err.Error())
t.Log(err)
} else {
assert.Equal(201, code, "Add adminjob status should be 201")
}
}
func TestGCGet(t *testing.T) {
assert := assert.New(t)
apiTest := newHarborAPI()
code, _, err := apiTest.GCScheduleGet(*admin)
if err != nil {
t.Error("Error occurred while get a admin job", err.Error())
t.Log(err)
} else {
assert.Equal(200, code, "Get adminjob status should be 200")
}
}

6
src/pkg/retention/controller.go
View File

@ -93,10 +93,11 @@ func (r *DefaultAPIController) CreateRetention(p *policy.Metadata) (int64, error
if p.Trigger.Kind == policy.TriggerKindSchedule {
cron, ok := p.Trigger.Settings[policy.TriggerSettingsCron]
if ok && len(cron.(string)) > 0 {
extras := make(map[string]interface{})
if _, err = r.scheduler.Schedule(orm.Context(), schedulerVendorType, id, "", cron.(string), SchedulerCallback, TriggerParam{
PolicyID: id,
Trigger: ExecutionTriggerSchedule,
}); err != nil {
}, extras); err != nil {
return 0, err
}
}
@ -152,10 +153,11 @@ func (r *DefaultAPIController) UpdateRetention(p *policy.Metadata) error {
}
}
if needSch {
extras := make(map[string]interface{})
_, err := r.scheduler.Schedule(orm.Context(), schedulerVendorType, p.ID, "", p.Trigger.Settings[policy.TriggerSettingsCron].(string), SchedulerCallback, TriggerParam{
PolicyID: p.ID,
Trigger: ExecutionTriggerSchedule,
})
}, extras)
if err != nil {
return err
}

2
src/pkg/retention/controller_test.go
View File

@ -220,7 +220,7 @@ func (s *ControllerTestSuite) TestExecution() {
type fakeRetentionScheduler struct {
}
func (f *fakeRetentionScheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}) (int64, error) {
func (f *fakeRetentionScheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
return 111, nil
}

1
src/pkg/scheduler/dao.go
View File

@ -34,6 +34,7 @@ type schedule struct {
VendorID int64 `orm:"column(vendor_id)"`
CRONType string `orm:"column(cron_type)"`
CRON string `orm:"column(cron)"`
ExtraAttrs string `orm:"column(extra_attrs)"`
CallbackFuncName string `orm:"column(callback_func_name)"`
CallbackFuncParam string `orm:"column(callback_func_param)"`
CreationTime time.Time `orm:"column(creation_time)"`

4
src/pkg/scheduler/dao_test.go
View File

@ -45,6 +45,7 @@ func (d *daoTestSuite) SetupTest() {
CRON: "0 * * * * *",
CallbackFuncName: "callback_func_01",
CallbackFuncParam: "callback_func_params",
ExtraAttrs: `{"key":"value"}`,
}
id, err := d.dao.Create(d.ctx, schedule)
d.Require().Nil(err)
@ -79,6 +80,7 @@ func (d *daoTestSuite) TestGet() {
schedule, err = d.dao.Get(d.ctx, d.id)
d.Require().Nil(err)
d.Equal(d.id, schedule.ID)
d.Equal("{\"key\":\"value\"}", schedule.ExtraAttrs)
}
func (d *daoTestSuite) TestDelete() {
@ -93,7 +95,7 @@ func (d *daoTestSuite) TestUpdate() {
// not found
err := d.dao.Update(d.ctx, &schedule{
ID: 10000,
})
}, "CRON")
d.True(errors.IsNotFoundErr(err))
// pass

41
src/pkg/scheduler/scheduler.go
View File

@ -37,14 +37,15 @@ var (
// Schedule describes the detail information about the created schedule
type Schedule struct {
ID int64 `json:"id"`
VendorType string `json:"vendor_type"`
VendorID int64 `json:"vendor_id"`
CRONType string `json:"cron_type"`
CRON string `json:"cron"`
Status string `json:"status"` // status of the underlying task(jobservice job)
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
ID int64 `json:"id"`
VendorType string `json:"vendor_type"`
VendorID int64 `json:"vendor_id"`
CRONType string `json:"cron_type"`
CRON string `json:"cron"`
ExtraAttrs map[string]interface{} `json:"extra_attrs"`
Status string `json:"status"` // status of the underlying task(jobservice job)
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
// we can extend this model to include more information(e.g. how many times the schedule already
// runs; when will the schedule runs next time)
}
@ -59,7 +60,7 @@ type Scheduler interface {
// The "params" is passed to the callback function as encoded json string, so the callback
// function must decode it before using
Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
cron string, callbackFuncName string, params interface{}) (int64, error)
cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error)
// UnScheduleByID the schedule specified by ID
UnScheduleByID(ctx context.Context, id int64) error
// UnScheduleByVendor the schedule specified by vendor
@ -94,10 +95,10 @@ type scheduler struct {
// to out of control from the global transaction, and uses a new transaction that only
// covers the logic inside the function
func (s *scheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
cron string, callbackFuncName string, params interface{}) (int64, error) {
cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
var scheduleID int64
f := func(ctx context.Context) error {
id, err := s.schedule(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params)
id, err := s.schedule(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
if err != nil {
return err
}
@ -113,7 +114,7 @@ func (s *scheduler) Schedule(ctx context.Context, vendorType string, vendorID in
}
func (s *scheduler) schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
cron string, callbackFuncName string, params interface{}) (int64, error) {
cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
if len(vendorType) == 0 {
return 0, fmt.Errorf("empty vendor type")
}
@ -142,6 +143,13 @@ func (s *scheduler) schedule(ctx context.Context, vendorType string, vendorID in
}
sched.CallbackFuncParam = string(paramsData)
}
if extras != nil {
extrasData, err := json.Marshal(extras)
if err != nil {
return 0, err
}
sched.ExtraAttrs = string(extrasData)
}
// create schedule record
// when checkin hook comes, the database record must exist,
// so the database record must be created first before submitting job
@ -272,6 +280,15 @@ func (s *scheduler) convertSchedule(ctx context.Context, schedule *schedule) (*S
CreationTime: schedule.CreationTime,
UpdateTime: schedule.UpdateTime,
}
if len(schedule.ExtraAttrs) > 0 {
extras := map[string]interface{}{}
if err := json.Unmarshal([]byte(schedule.ExtraAttrs), &extras); err != nil {
log.Errorf("failed to unmarshal the extra attributes of schedule %d: %v", schedule.ID, err)
return nil, err
}
schd.ExtraAttrs = extras
}
executions, err := s.execMgr.List(ctx, &q.Query{
Keywords: map[string]interface{}{
"VendorType": JobNameScheduler,

11
src/pkg/scheduler/scheduler_test.go
View File

@ -52,15 +52,16 @@ func (s *schedulerTestSuite) SetupTest() {
func (s *schedulerTestSuite) TestSchedule() {
// empty vendor type
id, err := s.scheduler.Schedule(nil, "", 0, "", "0 * * * * *", "callback", nil)
extras := make(map[string]interface{})
id, err := s.scheduler.Schedule(nil, "", 0, "", "0 * * * * *", "callback", nil, extras)
s.NotNil(err)
// invalid cron
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "", "callback", nil)
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "", "callback", nil, extras)
s.NotNil(err)
// callback function not exist
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "not-exist", nil)
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "not-exist", nil, extras)
s.NotNil(err)
// failed to submit to jobservice
@ -73,7 +74,7 @@ func (s *schedulerTestSuite) TestSchedule() {
Status: job.ErrorStatus.String(),
}, nil)
s.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
_, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param")
_, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param", extras)
s.Require().NotNil(err)
s.dao.AssertExpectations(s.T())
s.execMgr.AssertExpectations(s.T())
@ -91,7 +92,7 @@ func (s *schedulerTestSuite) TestSchedule() {
ExecutionID: 1,
Status: job.SuccessStatus.String(),
}, nil)
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param")
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param", extras)
s.Require().Nil(err)
s.Equal(int64(1), id)
s.dao.AssertExpectations(s.T())

16
src/pkg/task/model.go
View File

@ -115,6 +115,22 @@ func (t *Task) GetStringFromExtraAttrs(key string) string {
return str
}
// GetBoolFromExtraAttrs returns the bool value specified by key
func (t *Task) GetBoolFromExtraAttrs(key string) bool {
if len(t.ExtraAttrs) == 0 {
return false
}
rt, exist := t.ExtraAttrs[key]
if !exist {
return false
}
b, ok := rt.(bool)
if !ok {
return false
}
return b
}
// Job is the model represents the requested jobservice job
type Job struct {
Name string

2
src/portal/src/lib/components/config/gc/gc-history/gc-history.component.html
View File

@ -18,7 +18,7 @@
<clr-dg-cell>{{job.createTime | date:'medium'}}</clr-dg-cell>
<clr-dg-cell>{{job.updateTime | date:'medium'}}</clr-dg-cell>
<clr-dg-cell>
<a *ngIf="job.status.toLowerCase() === 'finished' || job.status.toLowerCase() === 'error'" target="_blank" [href]="getLogLink(job.id)"><clr-icon shape="list"></clr-icon></a>
<a *ngIf="job.status.toLowerCase() === 'success' || job.status.toLowerCase() === 'error'" target="_blank" [href]="getLogLink(job.id)"><clr-icon shape="list"></clr-icon></a>
</clr-dg-cell>
</clr-dg-row>
<clr-dg-footer>

6
src/replication/policy/controller/controller.go
View File

@ -52,7 +52,8 @@ func (c *controller) Create(policy *model.Policy) (int64, error) {
return 0, err
}
if isScheduledTrigger(policy) {
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, id, "", policy.Trigger.Settings.Cron, CallbackFuncName, id); err != nil {
extras := make(map[string]interface{})
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, id, "", policy.Trigger.Settings.Cron, CallbackFuncName, id, extras); err != nil {
log.Errorf("failed to schedule the policy %d: %v", id, err)
}
}
@ -84,7 +85,8 @@ func (c *controller) Update(policy *model.Policy) error {
}
// schedule again if needed
if isScheduledTrigger(policy) {
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, policy.ID, "", policy.Trigger.Settings.Cron, CallbackFuncName, policy.ID); err != nil {
extras := make(map[string]interface{})
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, policy.ID, "", policy.Trigger.Settings.Cron, CallbackFuncName, policy.ID, extras); err != nil {
return fmt.Errorf("failed to schedule the policy %d: %v", policy.ID, err)
}
}

4
src/replication/policy/controller/controller_test.go
View File

@ -226,7 +226,7 @@ func TestCreate(t *testing.T) {
// scheduled trigger
scheduler.On("Schedule", mock.Anything, mock.Anything,
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
_, err = ctl.Create(&model.Policy{
Enabled: true,
Trigger: &model.Trigger{
@ -269,7 +269,7 @@ func TestUpdate(t *testing.T) {
// the trigger changed
scheduler.On("Schedule", mock.Anything, mock.Anything,
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything,
mock.Anything).Return(nil)

198
src/server/v2.0/handler/gc.go Normal file
View File

@ -0,0 +1,198 @@
package handler
import (
"context"
"encoding/json"
"fmt"
"github.com/go-openapi/runtime/middleware"
"github.com/goharbor/harbor/src/controller/gc"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
"github.com/goharbor/harbor/src/server/v2.0/handler/model"
"github.com/goharbor/harbor/src/server/v2.0/models"
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/gc"
"os"
"strings"
)
type gcAPI struct {
BaseAPI
gcCtr gc.Controller
}
func newGCAPI() *gcAPI {
return &gcAPI{
gcCtr: gc.NewController(),
}
}
func (g *gcAPI) CreateGCSchedule(ctx context.Context, params operation.CreateGCScheduleParams) middleware.Responder {
id, err := g.kick(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters)
if err != nil {
return g.SendError(ctx, err)
}
// replace the /api/v2.0/system/gc/schedule/{id} to /api/v2.0/system/gc/{id}
lastSlashIndex := strings.LastIndex(params.HTTPRequest.URL.Path, "/")
if lastSlashIndex != -1 {
location := fmt.Sprintf("%s/%d", params.HTTPRequest.URL.Path[:lastSlashIndex], id)
return operation.NewCreateGCScheduleCreated().WithLocation(location)
}
return operation.NewCreateGCScheduleCreated()
}
func (g *gcAPI) UpdateGCSchedule(ctx context.Context, params operation.UpdateGCScheduleParams) middleware.Responder {
_, err := g.kick(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters)
if err != nil {
return g.SendError(ctx, err)
}
return operation.NewUpdateGCScheduleOK()
}
func (g *gcAPI) kick(ctx context.Context, scheType string, cron string, parameters map[string]interface{}) (int64, error) {
// set the required parameters for GC
parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
parameters["time_window"] = config.GetGCTimeWindow()
var err error
var id int64
switch scheType {
case ScheduleManual:
policy := gc.Policy{
ExtraAttrs: parameters,
}
if dryRun, ok := parameters["dry_run"].(bool); ok {
policy.DryRun = dryRun
}
if deleteUntagged, ok := parameters["delete_untagged"].(bool); ok {
policy.DeleteUntagged = deleteUntagged
}
id, err = g.gcCtr.Start(ctx, policy, task.ExecutionTriggerManual)
case ScheduleNone:
err = g.gcCtr.DeleteSchedule(ctx)
case ScheduleHourly, ScheduleDaily, ScheduleWeekly, ScheduleCustom:
policy := gc.Policy{
ExtraAttrs: parameters,
}
if dryRun, ok := parameters["dry_run"].(bool); ok {
policy.DryRun = dryRun
}
if deleteUntagged, ok := parameters["delete_untagged"].(bool); ok {
policy.DeleteUntagged = deleteUntagged
}
err = g.updateSchedule(ctx, scheType, cron, policy)
}
return id, err
}
func (g *gcAPI) createSchedule(ctx context.Context, cronType, cron string, policy gc.Policy) error {
if cron == "" {
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("empty cron string for gc schedule")
}
_, err := g.gcCtr.CreateSchedule(ctx, cronType, cron, policy)
if err != nil {
return err
}
return nil
}
func (g *gcAPI) updateSchedule(ctx context.Context, cronType, cron string, policy gc.Policy) error {
if err := g.gcCtr.DeleteSchedule(ctx); err != nil {
return err
}
return g.createSchedule(ctx, cronType, cron, policy)
}
func (g *gcAPI) GetGCSchedule(ctx context.Context, params operation.GetGCScheduleParams) middleware.Responder {
schedule, err := g.gcCtr.GetSchedule(ctx)
if errors.IsNotFoundErr(err) {
return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(&scheduler.Schedule{}).ToSwagger())
}
if err != nil {
return g.SendError(ctx, err)
}
return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(schedule).ToSwagger())
}
func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryParams) middleware.Responder {
query, err := g.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
if err != nil {
return g.SendError(ctx, err)
}
total, err := g.gcCtr.ExecutionCount(ctx, query)
if err != nil {
return g.SendError(ctx, err)
}
execs, err := g.gcCtr.ListExecutions(ctx, query)
if err != nil {
return g.SendError(ctx, err)
}
var hs []*model.GCHistory
for _, exec := range execs {
extraAttrsString, err := json.Marshal(exec.ExtraAttrs)
if err != nil {
return g.SendError(ctx, err)
}
hs = append(hs, &model.GCHistory{
ID: exec.ID,
Name: gc.GCVendorType,
Kind: exec.Trigger,
Parameters: string(extraAttrsString),
Schedule: &model.ScheduleParam{
Type: exec.Trigger,
},
Status: exec.Status,
CreationTime: exec.StartTime,
UpdateTime: exec.EndTime,
})
}
var results []*models.GCHistory
for _, h := range hs {
results = append(results, h.ToSwagger())
}
return operation.NewGetGCHistoryOK().
WithXTotalCount(total).
WithLink(g.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
WithPayload(results)
}
func (g *gcAPI) GetGC(ctx context.Context, params operation.GetGCParams) middleware.Responder {
exec, err := g.gcCtr.GetExecution(ctx, params.GcID)
if err != nil {
return g.SendError(ctx, err)
}
extraAttrsString, err := json.Marshal(exec.ExtraAttrs)
if err != nil {
return g.SendError(ctx, err)
}
res := &model.GCHistory{
ID: exec.ID,
Name: gc.GCVendorType,
Kind: exec.Trigger,
Parameters: string(extraAttrsString),
Status: exec.Status,
Schedule: &model.ScheduleParam{
Type: exec.Trigger,
},
CreationTime: exec.StartTime,
UpdateTime: exec.EndTime,
}
return operation.NewGetGCOK().WithPayload(res.ToSwagger())
}
func (g *gcAPI) GetGCLog(ctx context.Context, params operation.GetGCLogParams) middleware.Responder {
log, err := g.gcCtr.GetTaskLog(ctx, params.GcID)
if err != nil {
return g.SendError(ctx, err)
}
return operation.NewGetGCLogOK().WithPayload(string(log))
}

1
src/server/v2.0/handler/handler.go
View File

@ -41,6 +41,7 @@ func New() http.Handler {
ReplicationAPI: newReplicationAPI(),
SysteminfoAPI: newSystemInfoAPI(),
PingAPI: newPingAPI(),
GcAPI: newGCAPI(),
})
if err != nil {
log.Fatal(err)

84
src/server/v2.0/handler/model/gc.go Normal file
View File

@ -0,0 +1,84 @@
package model
import (
"encoding/json"
"github.com/go-openapi/strfmt"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/server/v2.0/models"
"time"
)
// ScheduleParam defines the parameter of schedule trigger
type ScheduleParam struct {
// Daily, Weekly, Custom, Manual, None
Type string `json:"type"`
// The cron string of scheduled job
Cron string `json:"cron"`
}
// GCHistory gc execution history
type GCHistory struct {
Schedule *ScheduleParam `json:"schedule"`
ID int64 `json:"id"`
Name string `json:"job_name"`
Kind string `json:"job_kind"`
Parameters string `json:"job_parameters"`
Status string `json:"job_status"`
UUID string `json:"-"`
Deleted bool `json:"deleted"`
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
}
// ToSwagger converts the history to the swagger model
func (h *GCHistory) ToSwagger() *models.GCHistory {
return &models.GCHistory{
ID: h.ID,
JobName: h.Name,
JobKind: h.Kind,
JobParameters: h.Parameters,
Deleted: h.Deleted,
JobStatus: h.Status,
Schedule: &models.ScheduleObj{
Cron: h.Schedule.Cron,
Type: h.Schedule.Type,
},
CreationTime: strfmt.DateTime(h.CreationTime),
UpdateTime: strfmt.DateTime(h.UpdateTime),
}
}
// Schedule ...
type Schedule struct {
*scheduler.Schedule
}
// ToSwagger converts the schedule to the swagger model
// TODO remove the hard code when after issue https://github.com/goharbor/harbor/issues/13047 is resolved.
func (s *Schedule) ToSwagger() *models.GCHistory {
e, err := json.Marshal(s.ExtraAttrs)
if err != nil {
log.Error(err)
}
return &models.GCHistory{
ID: 0,
JobName: "",
JobKind: s.CRON,
JobParameters: string(e),
Deleted: false,
JobStatus: "",
Schedule: &models.ScheduleObj{
Cron: s.CRON,
Type: "Custom",
},
CreationTime: strfmt.DateTime(s.CreationTime),
UpdateTime: strfmt.DateTime(s.UpdateTime),
}
}
// NewSchedule ...
func NewSchedule(s *scheduler.Schedule) *Schedule {
return &Schedule{Schedule: s}
}

15
src/server/v2.0/handler/util.go
View File

@ -29,6 +29,21 @@ import (
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
)
const (
// ScheduleHourly : 'Hourly'
ScheduleHourly = "Hourly"
// ScheduleDaily : 'Daily'
ScheduleDaily = "Daily"
// ScheduleWeekly : 'Weekly'
ScheduleWeekly = "Weekly"
// ScheduleCustom : 'Custom'
ScheduleCustom = "Custom"
// ScheduleManual : 'Manual'
ScheduleManual = "Manual"
// ScheduleNone : 'None'
ScheduleNone = "None"
)
func boolValue(v *bool) bool {
if v != nil {
return *v

4
src/server/v2.0/route/legacy.go
View File

@ -45,10 +45,6 @@ func registerLegacyRoutes() {
beego.Router("/api/"+version+"/quotas", &api.QuotaAPI{}, "get:List")
beego.Router("/api/"+version+"/quotas/:id([0-9]+)", &api.QuotaAPI{}, "get:Get;put:Put")
beego.Router("/api/"+version+"/system/gc", &api.GCAPI{}, "get:List")
beego.Router("/api/"+version+"/system/gc/:id", &api.GCAPI{}, "get:GetGC")
beego.Router("/api/"+version+"/system/gc/:id([0-9]+)/log", &api.GCAPI{}, "get:GetLog")
beego.Router("/api/"+version+"/system/gc/schedule", &api.GCAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/"+version+"/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/"+version+"/system/CVEAllowlist", &api.SysCVEAllowlistAPI{}, "get:Get;put:Put")
beego.Router("/api/"+version+"/system/oidc/ping", &api.OIDCAPI{}, "post:Ping")

14
src/testing/pkg/scheduler/scheduler.go
View File

@ -62,20 +62,20 @@ func (_m *Scheduler) ListSchedules(ctx context.Context, query *q.Query) ([]*sche
return r0, r1
}
// Schedule provides a mock function with given fields: ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params
func (_m *Scheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}) (int64, error) {
ret := _m.Called(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params)
// Schedule provides a mock function with given fields: ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras
func (_m *Scheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
ret := _m.Called(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, string, int64, string, string, string, interface{}) int64); ok {
r0 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params)
if rf, ok := ret.Get(0).(func(context.Context, string, int64, string, string, string, interface{}, map[string]interface{}) int64); ok {
r0 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string, int64, string, string, string, interface{}) error); ok {
r1 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params)
if rf, ok := ret.Get(1).(func(context.Context, string, int64, string, string, string, interface{}, map[string]interface{}) error); ok {
r1 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
} else {
r1 = ret.Error(1)
}

1
tests/apitests/python/library/base.py
View File

@ -61,6 +61,7 @@ def _create_client(server, credential, debug, api_type="products"):
"scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)),
"replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)),
"robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)),
"gc": v2_swagger_client.GcApi(v2_swagger_client.ApiClient(cfg)),
}.get(api_type,'Error: Wrong API type')
def _assert_status_code(expect_code, return_code):

134
tests/apitests/python/library/gc.py Normal file
View File

@ -0,0 +1,134 @@
# -*- coding: utf-8 -*-
import time
import base
import re
import v2_swagger_client
from v2_swagger_client.rest import ApiException
class GC(base.Base, object):
def __init__(self):
super(GC,self).__init__(api_type = "gc")
def get_gc_history(self, expect_status_code = 200, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
try:
data, status_code, _ = client.get_gc_history_with_http_info()
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return data
def get_gc_status_by_id(self, job_id, expect_status_code = 200, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
try:
data, status_code, _ = client.get_gc_with_http_info(job_id)
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return data
def get_gc_log_by_id(self, job_id, expect_status_code = 200, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
try:
data, status_code, _ = client.get_gc_log_with_http_info(job_id)
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return data
def get_gc_schedule(self, expect_status_code = 200, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
try:
data, status_code, _ = client.get_gc_schedule_with_http_info()
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return data
def create_gc_schedule(self, schedule_type, is_delete_untagged, cron = None, expect_status_code = 201, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
gc_parameters = {'delete_untagged':is_delete_untagged}
gc_schedule = v2_swagger_client.ScheduleObj()
gc_schedule.type = schedule_type
if cron is not None:
gc_schedule.cron = cron
gc_job = v2_swagger_client.Schedule()
gc_job.schedule = gc_schedule
gc_job.parameters = gc_parameters
try:
_, status_code, header = client.create_gc_schedule_with_http_info(gc_job)
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Create GC schedule response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Create GC schedule result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return base._get_id_from_header(header)
def gc_now(self, is_delete_untagged=False, **kwargs):
gc_id = self.create_gc_schedule('Manual', is_delete_untagged, **kwargs)
return gc_id
def validate_gc_job_status(self, gc_id, expected_gc_status, **kwargs):
get_gc_status_finish = False
timeout_count = 20
while timeout_count > 0:
time.sleep(5)
status = self.get_gc_status_by_id(gc_id, **kwargs)
print("GC job No: {}, status: {}".format(timeout_count, status.job_status))
if status.job_status == expected_gc_status:
get_gc_status_finish = True
break
timeout_count = timeout_count - 1
if not (get_gc_status_finish):
raise Exception("GC status is not as expected '{}' actual GC status is '{}'".format(expected_gc_status, status.job_status))
def validate_deletion_success(self, gc_id, **kwargs):
log_content = self.get_gc_log_by_id(gc_id, **kwargs)
key_message = "manifests eligible for deletion"
key_message_pos = log_content.find(key_message)
full_message = log_content[key_message_pos-30 : key_message_pos + len(key_message)]
deleted_files_count_list = re.findall(r'\s+(\d+)\s+blobs\s+and\s+\d+\s+manifests\s+eligible\s+for\s+deletion', full_message)
if len(deleted_files_count_list) != 1:
raise Exception(r"Fail to get blobs eligible for deletion in log file, failure is {}.".format(len(deleted_files_count_list)))
deleted_files_count = int(deleted_files_count_list[0])
if deleted_files_count == 0:
raise Exception(r"Get blobs eligible for deletion count is {}, while we expect more than 1.".format(deleted_files_count))

32
tests/apitests/python/library/system.py
View File

@ -157,38 +157,6 @@ class System(base.Base):
scan_all_id = self.create_scan_all_schedule('Manual', **kwargs)
return scan_all_id
def gc_now(self, is_delete_untagged=False, **kwargs):
gc_id = self.create_gc_schedule('Manual', is_delete_untagged, **kwargs)
return gc_id
def validate_gc_job_status(self, gc_id, expected_gc_status, **kwargs):
get_gc_status_finish = False
timeout_count = 20
while timeout_count > 0:
time.sleep(5)
status = self.get_gc_status_by_id(gc_id, **kwargs)
print("GC job No: {}, status: {}".format(timeout_count, status.job_status))
if status.job_status == expected_gc_status:
get_gc_status_finish = True
break
timeout_count = timeout_count - 1
if not (get_gc_status_finish):
raise Exception("GC status is not as expected '{}' actual GC status is '{}'".format(expected_gc_status, status.job_status))
def validate_deletion_success(self, gc_id, **kwargs):
log_content = self.get_gc_log_by_id(gc_id, **kwargs)
key_message = "manifests eligible for deletion"
key_message_pos = log_content.find(key_message)
full_message = log_content[key_message_pos-30 : key_message_pos + len(key_message)]
deleted_files_count_list = re.findall(r'\s+(\d+)\s+blobs\s+and\s+\d+\s+manifests\s+eligible\s+for\s+deletion', full_message)
if len(deleted_files_count_list) != 1:
raise Exception(r"Fail to get blobs eligible for deletion in log file, failure is {}.".format(len(deleted_files_count_list)))
deleted_files_count = int(deleted_files_count_list[0])
if deleted_files_count == 0:
raise Exception(r"Get blobs eligible for deletion count is {}, while we expect more than 1.".format(deleted_files_count))
def set_cve_allowlist(self, expires_at=None, expected_status_code=200, *cve_ids, **kwargs):
client = self._get_client(**kwargs)
cve_list = [swagger_client.CVEAllowlistItem(cve_id=c) for c in cve_ids]

16
tests/apitests/python/test_garbage_collection.py
View File

@ -7,17 +7,17 @@ from testutils import ADMIN_CLIENT, suppress_urllib3_warning
from testutils import TEARDOWN
from testutils import harbor_server
from library.user import User
from library.system import System
from library.project import Project
from library.repository import Repository
from library.base import _assert_status_code
from library.repository import push_special_image_to_project
from library.artifact import Artifact
from library.gc import GC
class TestProjects(unittest.TestCase):
@suppress_urllib3_warning
def setUp(self):
self.system = System()
self.gc = GC()
self.project = Project()
self.user = User()
self.repo = Repository()
@ -82,13 +82,13 @@ class TestProjects(unittest.TestCase):
self.artifact.delete_tag(TestProjects.project_gc_untag_name, self.repo_name_untag, self.tag, self.tag, **ADMIN_CLIENT)
#5. Tigger garbage collection operation;
gc_id = self.system.gc_now(**ADMIN_CLIENT)
gc_id = self.gc.gc_now(**ADMIN_CLIENT)
#6. Check garbage collection job was finished;
self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT)
self.gc.validate_gc_job_status(gc_id, "Success", **ADMIN_CLIENT)
#7. Get garbage collection log, check there is a number of files was deleted;
self.system.validate_deletion_success(gc_id, **ADMIN_CLIENT)
self.gc.validate_deletion_success(gc_id, **ADMIN_CLIENT)
artifacts = self.artifact.list_artifacts(TestProjects.project_gc_untag_name, self.repo_name_untag, **TestProjects.USER_GC_CLIENT)
_assert_status_code(len(artifacts), 1)
@ -96,13 +96,13 @@ class TestProjects(unittest.TestCase):
time.sleep(5)
#9. Tigger garbage collection operation;
gc_id = self.system.gc_now(is_delete_untagged=True, **ADMIN_CLIENT)
gc_id = self.gc.gc_now(is_delete_untagged=True, **ADMIN_CLIENT)
#10. Check garbage collection job was finished;
self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT)
self.gc.validate_gc_job_status(gc_id, "Success", **ADMIN_CLIENT)
#7. Get garbage collection log, check there is a number of files was deleted;
self.system.validate_deletion_success(gc_id, **ADMIN_CLIENT)
self.gc.validate_deletion_success(gc_id, **ADMIN_CLIENT)
#11. Repository with untag image should be still there;
repo_data_untag = self.repo.list_repositories(TestProjects.project_gc_untag_name, **TestProjects.USER_GC_CLIENT)