mirror of
https://github.com/goharbor/harbor.git
synced 2025-01-11 10:27:58 +01:00
Merge pull request #13139 from wy65701436/migrate-gc
Migrate gc to task manager
This commit is contained in:
commit
6569016d35
@ -1549,25 +1549,6 @@ paths:
|
||||
description: Only admin has this authority.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
/system/gc:
|
||||
get:
|
||||
summary: Get gc results.
|
||||
description: This endpoint let user get latest ten gc results.
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'200':
|
||||
description: Get gc results successfully.
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/definitions/GCResult'
|
||||
'401':
|
||||
description: User need to log in first.
|
||||
'403':
|
||||
description: User does not have permission of admin role.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
'/system/gc/{id}':
|
||||
get:
|
||||
summary: Get gc status.
|
||||
@ -1592,101 +1573,6 @@ paths:
|
||||
description: User does not have permission of admin role.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
'/system/gc/{id}/log':
|
||||
get:
|
||||
summary: Get gc job log.
|
||||
description: This endpoint let user get gc job logs filtered by specific ID.
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
type: integer
|
||||
format: int64
|
||||
required: true
|
||||
description: Relevant job ID
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'200':
|
||||
description: Get successfully.
|
||||
schema:
|
||||
type: string
|
||||
'400':
|
||||
description: Illegal format of provided ID value.
|
||||
'401':
|
||||
description: User need to log in first.
|
||||
'403':
|
||||
description: User does not have permission of admin role.
|
||||
'404':
|
||||
description: The specific gc ID's log does not exist.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
/system/gc/schedule:
|
||||
get:
|
||||
summary: Get gc's schedule.
|
||||
description: This endpoint is for get schedule of gc job.
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'200':
|
||||
description: Get gc's schedule.
|
||||
schema:
|
||||
$ref: '#/definitions/AdminJobSchedule'
|
||||
'401':
|
||||
description: User need to log in first.
|
||||
'403':
|
||||
description: Only admin has this authority.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
put:
|
||||
summary: Update gc's schedule.
|
||||
description: |
|
||||
This endpoint is for update gc schedule.
|
||||
parameters:
|
||||
- name: schedule
|
||||
in: body
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/AdminJobSchedule'
|
||||
description: Updates of gc's schedule.
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'200':
|
||||
description: Updated gc's schedule successfully.
|
||||
'400':
|
||||
description: Invalid schedule type.
|
||||
'401':
|
||||
description: User need to log in first.
|
||||
'403':
|
||||
description: User does not have permission of admin role.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
post:
|
||||
summary: Create a gc schedule.
|
||||
description: |
|
||||
This endpoint is for update gc schedule.
|
||||
parameters:
|
||||
- name: schedule
|
||||
in: body
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/AdminJobSchedule'
|
||||
description: Updates of gc's schedule.
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'200':
|
||||
description: GC schedule successfully.
|
||||
'400':
|
||||
description: Invalid schedule type.
|
||||
'401':
|
||||
description: User need to log in first.
|
||||
'403':
|
||||
description: User does not have permission of admin role.
|
||||
'409':
|
||||
description: There is a "gc" job in progress, so the request cannot be served.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
/system/scanAll/schedule:
|
||||
get:
|
||||
summary: Get scan_all's schedule.
|
||||
|
@ -2017,6 +2017,155 @@ paths:
|
||||
description: Not found the default root certificate.
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
/system/gc:
|
||||
get:
|
||||
summary: Get gc results.
|
||||
description: This endpoint let user get gc execution history.
|
||||
tags:
|
||||
- gc
|
||||
operationId: getGCHistory
|
||||
parameters:
|
||||
- $ref: '#/parameters/query'
|
||||
- $ref: '#/parameters/page'
|
||||
- $ref: '#/parameters/pageSize'
|
||||
responses:
|
||||
'200':
|
||||
description: Get gc results successfully.
|
||||
headers:
|
||||
X-Total-Count:
|
||||
description: The total count of history
|
||||
type: integer
|
||||
Link:
|
||||
description: Link refers to the previous page and next page
|
||||
type: string
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/definitions/GCHistory'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
/system/gc/{gc_id}:
|
||||
get:
|
||||
summary: Get gc status.
|
||||
description: This endpoint let user get gc status filtered by specific ID.
|
||||
operationId: getGC
|
||||
parameters:
|
||||
- $ref: '#/parameters/gcId'
|
||||
tags:
|
||||
- gc
|
||||
responses:
|
||||
'200':
|
||||
description: Get gc results successfully.
|
||||
schema:
|
||||
$ref: '#/definitions/GCHistory'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'404':
|
||||
$ref: '#/responses/404'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
/system/gc/{gc_id}/log:
|
||||
get:
|
||||
summary: Get gc job log.
|
||||
description: This endpoint let user get gc job logs filtered by specific ID.
|
||||
operationId: getGCLog
|
||||
parameters:
|
||||
- $ref: '#/parameters/gcId'
|
||||
tags:
|
||||
- gc
|
||||
produces:
|
||||
- text/plain
|
||||
responses:
|
||||
'200':
|
||||
description: Get successfully.
|
||||
schema:
|
||||
type: string
|
||||
'400':
|
||||
$ref: '#/responses/400'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'404':
|
||||
$ref: '#/responses/404'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
/system/gc/schedule:
|
||||
get:
|
||||
summary: Get gc's schedule.
|
||||
description: This endpoint is for get schedule of gc job.
|
||||
operationId: getGCSchedule
|
||||
tags:
|
||||
- gc
|
||||
responses:
|
||||
'200':
|
||||
description: Get gc's schedule.
|
||||
schema:
|
||||
$ref: '#/definitions/GCHistory'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
post:
|
||||
summary: Create a gc schedule.
|
||||
description: |
|
||||
This endpoint is for update gc schedule.
|
||||
operationId: createGCSchedule
|
||||
parameters:
|
||||
- name: schedule
|
||||
in: body
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/Schedule'
|
||||
description: Updates of gc's schedule.
|
||||
tags:
|
||||
- gc
|
||||
responses:
|
||||
'201':
|
||||
$ref: '#/responses/201'
|
||||
'400':
|
||||
$ref: '#/responses/400'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'409':
|
||||
$ref: '#/responses/409'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
put:
|
||||
summary: Update gc's schedule.
|
||||
description: |
|
||||
This endpoint is for update gc schedule.
|
||||
operationId: updateGCSchedule
|
||||
parameters:
|
||||
- name: schedule
|
||||
in: body
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/Schedule'
|
||||
description: Updates of gc's schedule.
|
||||
tags:
|
||||
- gc
|
||||
responses:
|
||||
'200':
|
||||
description: Updated gc's schedule successfully.
|
||||
'400':
|
||||
$ref: '#/responses/400'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
/ping:
|
||||
get:
|
||||
summary: Ping Harbor to check if it's alive.
|
||||
@ -2134,6 +2283,13 @@ parameters:
|
||||
description: Robot ID
|
||||
required: true
|
||||
type: integer
|
||||
gcId:
|
||||
name: gc_id
|
||||
in: path
|
||||
description: The ID of the gc log
|
||||
required: true
|
||||
type: integer
|
||||
format: int64
|
||||
responses:
|
||||
'200':
|
||||
description: Success
|
||||
@ -3351,3 +3507,55 @@ definitions:
|
||||
description: The storage of system.
|
||||
items:
|
||||
$ref: '#/definitions/Storage'
|
||||
GCHistory:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
type: integer
|
||||
description: the id of gc job.
|
||||
job_name:
|
||||
type: string
|
||||
description: the job name of gc job.
|
||||
job_kind:
|
||||
type: string
|
||||
description: the job kind of gc job.
|
||||
job_parameters:
|
||||
type: string
|
||||
description: the job parameters of gc job.
|
||||
schedule:
|
||||
$ref: '#/definitions/ScheduleObj'
|
||||
job_status:
|
||||
type: string
|
||||
description: the status of gc job.
|
||||
deleted:
|
||||
type: boolean
|
||||
description: if gc job was deleted.
|
||||
creation_time:
|
||||
type: string
|
||||
format: date-time
|
||||
description: the creation time of gc job.
|
||||
update_time:
|
||||
type: string
|
||||
format: date-time
|
||||
description: the update time of gc job.
|
||||
Schedule:
|
||||
type: object
|
||||
properties:
|
||||
schedule:
|
||||
$ref: '#/definitions/ScheduleObj'
|
||||
parameters:
|
||||
type: object
|
||||
description: The parameters of admin job
|
||||
additionalProperties:
|
||||
type: object
|
||||
ScheduleObj:
|
||||
type: object
|
||||
properties:
|
||||
type:
|
||||
type: string
|
||||
description: |
|
||||
The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manually' and 'None'.
|
||||
'Manually' means to trigger it right away and 'None' means to cancel the schedule.
|
||||
cron:
|
||||
type: string
|
||||
description: A cron expression, a time-based job scheduler.
|
||||
|
@ -1,4 +1,4 @@
|
||||
/*
|
||||
/*
|
||||
Fixes issue https://github.com/goharbor/harbor/issues/13317
|
||||
Ensure the role_id of maintainer is 4 and the role_id of limisted guest is 5
|
||||
*/
|
||||
@ -268,3 +268,7 @@ BEGIN
|
||||
UPDATE scanner_registration SET is_default = TRUE WHERE name = 'Trivy' AND immutable = TRUE;
|
||||
END IF;
|
||||
END $$;
|
||||
ALTER TABLE execution ALTER COLUMN vendor_type type varchar(64);
|
||||
ALTER TABLE schedule ALTER COLUMN vendor_type type varchar(64);
|
||||
ALTER TABLE schedule ADD COLUMN IF NOT EXISTS extra_attrs JSON;
|
||||
ALTER TABLE task ALTER COLUMN vendor_type type varchar(64);
|
||||
|
27
src/controller/gc/callback.go
Normal file
27
src/controller/gc/callback.go
Normal file
@ -0,0 +1,27 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/lib/orm"
|
||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
)
|
||||
|
||||
func init() {
|
||||
err := scheduler.RegisterCallbackFunc(SchedulerCallback, gcCallback)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to registry GC call back, %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func gcCallback(ctx context.Context, p string) error {
|
||||
param := &Policy{}
|
||||
if err := json.Unmarshal([]byte(p), param); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal the param: %v", err)
|
||||
}
|
||||
_, err := Ctl.Start(orm.Context(), *param, task.ExecutionTriggerSchedule)
|
||||
return err
|
||||
}
|
217
src/controller/gc/controller.go
Normal file
217
src/controller/gc/controller.go
Normal file
@ -0,0 +1,217 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/errors"
|
||||
"github.com/goharbor/harbor/src/lib/q"
|
||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
)
|
||||
|
||||
var (
|
||||
// Ctl is a global garbage collection controller instance
|
||||
Ctl = NewController()
|
||||
)
|
||||
|
||||
const (
|
||||
// SchedulerCallback ...
|
||||
SchedulerCallback = "GARBAGE_COLLECTION"
|
||||
// GCVendorType ...
|
||||
GCVendorType = "GARBAGE_COLLECTION"
|
||||
)
|
||||
|
||||
// Controller manages the tags
|
||||
type Controller interface {
|
||||
// Start start a manual gc job
|
||||
Start(ctx context.Context, policy Policy, trigger string) (int64, error)
|
||||
// Stop stop a gc job
|
||||
Stop(ctx context.Context, id int64) error
|
||||
|
||||
// ExecutionCount returns the total count of executions according to the query
|
||||
ExecutionCount(ctx context.Context, query *q.Query) (count int64, err error)
|
||||
// ListExecutions lists the executions according to the query
|
||||
ListExecutions(ctx context.Context, query *q.Query) (executions []*Execution, err error)
|
||||
// GetExecution gets the specific execution
|
||||
GetExecution(ctx context.Context, executionID int64) (execution *Execution, err error)
|
||||
|
||||
// GetTask gets the specific task
|
||||
GetTask(ctx context.Context, id int64) (*Task, error)
|
||||
// GetTaskLog gets log of the specific task
|
||||
GetTaskLog(ctx context.Context, id int64) ([]byte, error)
|
||||
|
||||
// GetSchedule get the current gc schedule
|
||||
GetSchedule(ctx context.Context) (*scheduler.Schedule, error)
|
||||
// CreateSchedule create the gc schedule with cron type & string
|
||||
CreateSchedule(ctx context.Context, cronType, cron string, policy Policy) (int64, error)
|
||||
// DeleteSchedule remove the gc schedule
|
||||
DeleteSchedule(ctx context.Context) error
|
||||
}
|
||||
|
||||
// NewController creates an instance of the default repository controller
|
||||
func NewController() Controller {
|
||||
return &controller{
|
||||
taskMgr: task.NewManager(),
|
||||
exeMgr: task.NewExecutionManager(),
|
||||
schedulerMgr: scheduler.New(),
|
||||
}
|
||||
}
|
||||
|
||||
type controller struct {
|
||||
taskMgr task.Manager
|
||||
exeMgr task.ExecutionManager
|
||||
schedulerMgr scheduler.Scheduler
|
||||
}
|
||||
|
||||
// Start starts the manual GC
|
||||
func (c *controller) Start(ctx context.Context, policy Policy, trigger string) (int64, error) {
|
||||
para := make(map[string]interface{})
|
||||
para["delete_untagged"] = policy.DeleteUntagged
|
||||
para["dry_run"] = policy.DryRun
|
||||
para["redis_url_reg"] = policy.ExtraAttrs["redis_url_reg"]
|
||||
para["time_window"] = policy.ExtraAttrs["time_window"]
|
||||
|
||||
execID, err := c.exeMgr.Create(ctx, GCVendorType, -1, trigger, para)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
_, err = c.taskMgr.Create(ctx, execID, &task.Job{
|
||||
Name: job.ImageGC,
|
||||
Metadata: &job.Metadata{
|
||||
JobKind: job.KindGeneric,
|
||||
},
|
||||
Parameters: para,
|
||||
})
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
return execID, nil
|
||||
}
|
||||
|
||||
// Stop ...
|
||||
func (c *controller) Stop(ctx context.Context, id int64) error {
|
||||
return c.exeMgr.Stop(ctx, id)
|
||||
}
|
||||
|
||||
// ExecutionCount ...
|
||||
func (c *controller) ExecutionCount(ctx context.Context, query *q.Query) (int64, error) {
|
||||
query.Keywords["VendorType"] = GCVendorType
|
||||
return c.exeMgr.Count(ctx, query)
|
||||
}
|
||||
|
||||
// ListExecutions ...
|
||||
func (c *controller) ListExecutions(ctx context.Context, query *q.Query) ([]*Execution, error) {
|
||||
query = q.MustClone(query)
|
||||
query.Keywords["VendorType"] = GCVendorType
|
||||
|
||||
execs, err := c.exeMgr.List(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var executions []*Execution
|
||||
for _, exec := range execs {
|
||||
executions = append(executions, convertExecution(exec))
|
||||
}
|
||||
return executions, nil
|
||||
}
|
||||
|
||||
// GetExecution ...
|
||||
func (c *controller) GetExecution(ctx context.Context, id int64) (*Execution, error) {
|
||||
execs, err := c.exeMgr.List(ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"ID": id,
|
||||
"VendorType": GCVendorType,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(execs) == 0 {
|
||||
return nil, errors.New(nil).WithCode(errors.NotFoundCode).
|
||||
WithMessage("garbage collection execution %d not found", id)
|
||||
}
|
||||
return convertExecution(execs[0]), nil
|
||||
}
|
||||
|
||||
// GetTask ...
|
||||
func (c *controller) GetTask(ctx context.Context, id int64) (*Task, error) {
|
||||
tasks, err := c.taskMgr.List(ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"ID": id,
|
||||
"VendorType": GCVendorType,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return nil, errors.New(nil).WithCode(errors.NotFoundCode).
|
||||
WithMessage("garbage collection task %d not found", id)
|
||||
}
|
||||
return convertTask(tasks[0]), nil
|
||||
}
|
||||
|
||||
// GetTaskLog ...
|
||||
func (c *controller) GetTaskLog(ctx context.Context, id int64) ([]byte, error) {
|
||||
_, err := c.GetTask(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.taskMgr.GetLog(ctx, id)
|
||||
}
|
||||
|
||||
// GetSchedule ...
|
||||
func (c *controller) GetSchedule(ctx context.Context) (*scheduler.Schedule, error) {
|
||||
sch, err := c.schedulerMgr.ListSchedules(ctx, q.New(q.KeyWords{"VendorType": GCVendorType}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(sch) == 0 {
|
||||
return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc schedule is found")
|
||||
}
|
||||
if sch[0] == nil {
|
||||
return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc schedule is found")
|
||||
}
|
||||
return sch[0], nil
|
||||
}
|
||||
|
||||
// CreateSchedule ...
|
||||
func (c *controller) CreateSchedule(ctx context.Context, cronType, cron string, policy Policy) (int64, error) {
|
||||
extras := make(map[string]interface{})
|
||||
extras["delete_untagged"] = policy.DeleteUntagged
|
||||
return c.schedulerMgr.Schedule(ctx, GCVendorType, -1, cronType, cron, SchedulerCallback, policy, extras)
|
||||
}
|
||||
|
||||
// DeleteSchedule ...
|
||||
func (c *controller) DeleteSchedule(ctx context.Context) error {
|
||||
return c.schedulerMgr.UnScheduleByVendor(ctx, GCVendorType, -1)
|
||||
}
|
||||
|
||||
func convertExecution(exec *task.Execution) *Execution {
|
||||
return &Execution{
|
||||
ID: exec.ID,
|
||||
Status: exec.Status,
|
||||
StatusMessage: exec.StatusMessage,
|
||||
Trigger: exec.Trigger,
|
||||
ExtraAttrs: exec.ExtraAttrs,
|
||||
StartTime: exec.StartTime,
|
||||
EndTime: exec.EndTime,
|
||||
}
|
||||
}
|
||||
|
||||
func convertTask(task *task.Task) *Task {
|
||||
return &Task{
|
||||
ID: task.ID,
|
||||
ExecutionID: task.ExecutionID,
|
||||
Status: task.Status,
|
||||
StatusMessage: task.StatusMessage,
|
||||
RunCount: task.RunCount,
|
||||
DeleteUntagged: task.GetBoolFromExtraAttrs("delete_untagged"),
|
||||
DryRun: task.GetBoolFromExtraAttrs("dry_run"),
|
||||
JobID: task.JobID,
|
||||
CreationTime: task.CreationTime,
|
||||
StartTime: task.StartTime,
|
||||
UpdateTime: task.UpdateTime,
|
||||
EndTime: task.EndTime,
|
||||
}
|
||||
}
|
148
src/controller/gc/controller_test.go
Normal file
148
src/controller/gc/controller_test.go
Normal file
@ -0,0 +1,148 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/q"
|
||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
"github.com/goharbor/harbor/src/testing/mock"
|
||||
schedulertesting "github.com/goharbor/harbor/src/testing/pkg/scheduler"
|
||||
tasktesting "github.com/goharbor/harbor/src/testing/pkg/task"
|
||||
"github.com/stretchr/testify/suite"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type gcCtrTestSuite struct {
|
||||
suite.Suite
|
||||
scheduler *schedulertesting.Scheduler
|
||||
execMgr *tasktesting.ExecutionManager
|
||||
taskMgr *tasktesting.Manager
|
||||
ctl *controller
|
||||
}
|
||||
|
||||
func (g *gcCtrTestSuite) SetupTest() {
|
||||
g.execMgr = &tasktesting.ExecutionManager{}
|
||||
g.taskMgr = &tasktesting.Manager{}
|
||||
g.scheduler = &schedulertesting.Scheduler{}
|
||||
g.ctl = &controller{
|
||||
taskMgr: g.taskMgr,
|
||||
exeMgr: g.execMgr,
|
||||
schedulerMgr: g.scheduler,
|
||||
}
|
||||
}
|
||||
|
||||
func (g *gcCtrTestSuite) TestStart() {
|
||||
g.execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
g.taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
g.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
|
||||
|
||||
dataMap := make(map[string]interface{})
|
||||
p := Policy{
|
||||
DeleteUntagged: true,
|
||||
ExtraAttrs: dataMap,
|
||||
}
|
||||
id, err := g.ctl.Start(nil, p, task.ExecutionTriggerManual)
|
||||
g.Nil(err)
|
||||
g.Equal(int64(1), id)
|
||||
}
|
||||
|
||||
func (g *gcCtrTestSuite) TestStop() {
|
||||
g.execMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
|
||||
g.Nil(g.ctl.Stop(nil, 1))
|
||||
}
|
||||
|
||||
func (g *gcCtrTestSuite) TestGetTaskLog() {
|
||||
g.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
|
||||
{
|
||||
ID: 1,
|
||||
ExecutionID: 1,
|
||||
Status: job.SuccessStatus.String(),
|
||||
},
|
||||
}, nil)
|
||||
g.taskMgr.On("GetLog", mock.Anything, mock.Anything).Return([]byte("hello world"), nil)
|
||||
|
||||
log, err := g.ctl.GetTaskLog(nil, 1)
|
||||
g.Nil(err)
|
||||
g.Equal([]byte("hello world"), log)
|
||||
}
|
||||
|
||||
func (g *gcCtrTestSuite) TestExecutionCount() {
|
||||
g.execMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
count, err := g.ctl.ExecutionCount(nil, q.New(q.KeyWords{"VendorType": "gc"}))
|
||||
g.Nil(err)
|
||||
g.Equal(int64(1), count)
|
||||
}
|
||||
|
||||
func (g *gcCtrTestSuite) TestGetExecution() {
|
||||
g.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
|
||||
{
|
||||
ID: 1,
|
||||
Trigger: "Manual",
|
||||
VendorType: GCVendorType,
|
||||
StatusMessage: "Success",
|
||||
},
|
||||
}, nil)
|
||||
|
||||
hs, err := g.ctl.GetExecution(nil, int64(1))
|
||||
g.Nil(err)
|
||||
|
||||
g.Equal("Manual", hs.Trigger)
|
||||
}
|
||||
|
||||
func (g *gcCtrTestSuite) TestListExecutions() {
|
||||
g.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
|
||||
{
|
||||
ID: 1,
|
||||
Trigger: "Manual",
|
||||
},
|
||||
}, nil)
|
||||
|
||||
g.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
|
||||
{
|
||||
ID: 112,
|
||||
ExecutionID: 1,
|
||||
Status: job.SuccessStatus.String(),
|
||||
},
|
||||
}, nil)
|
||||
|
||||
hs, err := g.ctl.ListExecutions(nil, q.New(q.KeyWords{"VendorType": "gc"}))
|
||||
|
||||
g.Nil(err)
|
||||
g.Equal("Manual", hs[0].Trigger)
|
||||
}
|
||||
|
||||
func (g *gcCtrTestSuite) TestGetSchedule() {
|
||||
g.scheduler.On("ListSchedules", mock.Anything, mock.Anything).Return([]*scheduler.Schedule{
|
||||
{
|
||||
ID: 1,
|
||||
VendorType: "gc",
|
||||
},
|
||||
}, nil)
|
||||
|
||||
sche, err := g.ctl.GetSchedule(nil)
|
||||
g.Nil(err)
|
||||
g.Equal("gc", sche.VendorType)
|
||||
}
|
||||
|
||||
func (g *gcCtrTestSuite) TestCreateSchedule() {
|
||||
g.scheduler.On("Schedule", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
|
||||
mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
|
||||
dataMap := make(map[string]interface{})
|
||||
p := Policy{
|
||||
DeleteUntagged: true,
|
||||
ExtraAttrs: dataMap,
|
||||
}
|
||||
id, err := g.ctl.CreateSchedule(nil, "Daily", "* * * * * *", p)
|
||||
g.Nil(err)
|
||||
g.Equal(int64(1), id)
|
||||
}
|
||||
|
||||
func (g *gcCtrTestSuite) TestDeleteSchedule() {
|
||||
g.scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||
g.Nil(g.ctl.DeleteSchedule(nil))
|
||||
}
|
||||
|
||||
func TestControllerTestSuite(t *testing.T) {
|
||||
suite.Run(t, &gcCtrTestSuite{})
|
||||
}
|
54
src/controller/gc/model.go
Normal file
54
src/controller/gc/model.go
Normal file
@ -0,0 +1,54 @@
|
||||
package gc
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Policy ...
|
||||
type Policy struct {
|
||||
Trigger *Trigger `json:"trigger"`
|
||||
DeleteUntagged bool `json:"bool"`
|
||||
DryRun bool `json:"dryrun"`
|
||||
ExtraAttrs map[string]interface{} `json:"extra_attrs"`
|
||||
}
|
||||
|
||||
// TriggerType represents the type of trigger.
|
||||
type TriggerType string
|
||||
|
||||
// Trigger holds info for a trigger
|
||||
type Trigger struct {
|
||||
Type TriggerType `json:"type"`
|
||||
Settings *TriggerSettings `json:"trigger_settings"`
|
||||
}
|
||||
|
||||
// TriggerSettings is the setting about the trigger
|
||||
type TriggerSettings struct {
|
||||
Cron string `json:"cron"`
|
||||
}
|
||||
|
||||
// Execution model for replication
|
||||
type Execution struct {
|
||||
ID int64
|
||||
Status string
|
||||
StatusMessage string
|
||||
Trigger string
|
||||
ExtraAttrs map[string]interface{}
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
}
|
||||
|
||||
// Task model for replication
|
||||
type Task struct {
|
||||
ID int64
|
||||
ExecutionID int64
|
||||
Status string
|
||||
StatusMessage string
|
||||
RunCount int32
|
||||
DeleteUntagged bool
|
||||
DryRun bool
|
||||
JobID string
|
||||
CreationTime time.Time
|
||||
StartTime time.Time
|
||||
UpdateTime time.Time
|
||||
EndTime time.Time
|
||||
}
|
@ -292,8 +292,9 @@ func (c *controller) CreatePolicy(ctx context.Context, schema *policyModels.Sche
|
||||
schema.Trigger.Type == policyModels.TriggerTypeScheduled &&
|
||||
len(schema.Trigger.Settings.Cron) > 0 {
|
||||
// schedule and update policy
|
||||
extras := make(map[string]interface{})
|
||||
if _, err = c.scheduler.Schedule(ctx, job.P2PPreheat, id, "", schema.Trigger.Settings.Cron,
|
||||
SchedulerCallback, TriggerParam{PolicyID: id}); err != nil {
|
||||
SchedulerCallback, TriggerParam{PolicyID: id}, extras); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
@ -384,8 +385,9 @@ func (c *controller) UpdatePolicy(ctx context.Context, schema *policyModels.Sche
|
||||
|
||||
// schedule new
|
||||
if needSch {
|
||||
extras := make(map[string]interface{})
|
||||
if _, err := c.scheduler.Schedule(ctx, job.P2PPreheat, schema.ID, "", cron, SchedulerCallback,
|
||||
TriggerParam{PolicyID: schema.ID}); err != nil {
|
||||
TriggerParam{PolicyID: schema.ID}, extras); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
@ -241,7 +241,7 @@ func (s *preheatSuite) TestCreatePolicy() {
|
||||
FiltersStr: `[{"type":"repository","value":"harbor*"},{"type":"tag","value":"2*"}]`,
|
||||
TriggerStr: fmt.Sprintf(`{"type":"%s", "trigger_setting":{"cron":"* * * * */1"}}`, policy.TriggerTypeScheduled),
|
||||
}
|
||||
s.fakeScheduler.On("Schedule", s.ctx, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
s.fakeScheduler.On("Schedule", s.ctx, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
s.fakePolicyMgr.On("Create", s.ctx, policy).Return(int64(1), nil)
|
||||
s.fakePolicyMgr.On("Update", s.ctx, mock.Anything, mock.Anything).Return(nil)
|
||||
s.fakeScheduler.On("UnScheduleByVendor", s.ctx, mock.Anything, mock.Anything).Return(nil)
|
||||
|
@ -122,9 +122,6 @@ func init() {
|
||||
beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping")
|
||||
beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List")
|
||||
beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete")
|
||||
beego.Router("/api/system/gc/:id", &GCAPI{}, "get:GetGC")
|
||||
beego.Router("/api/system/gc/:id([0-9]+)/log", &GCAPI{}, "get:GetLog")
|
||||
beego.Router("/api/system/gc/schedule", &GCAPI{}, "get:Get;put:Put;post:Post")
|
||||
beego.Router("/api/system/scanAll/schedule", &ScanAllAPI{}, "get:Get;put:Put;post:Post")
|
||||
beego.Router("/api/system/CVEAllowlist", &SysCVEAllowlistAPI{}, "get:Get;put:Put")
|
||||
beego.Router("/api/system/oidc/ping", &OIDCAPI{}, "post:Ping")
|
||||
@ -864,36 +861,6 @@ func (a testapi) DeleteMeta(authInfor usrInfo, projectID int64, name string) (in
|
||||
return code, string(body), err
|
||||
}
|
||||
|
||||
func (a testapi) AddGC(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) {
|
||||
_sling := sling.New().Post(a.basePath)
|
||||
|
||||
path := "/api/system/gc/schedule"
|
||||
|
||||
_sling = _sling.Path(path)
|
||||
|
||||
// body params
|
||||
_sling = _sling.BodyJSON(adminReq)
|
||||
var httpStatusCode int
|
||||
var err error
|
||||
|
||||
httpStatusCode, _, err = request(_sling, jsonAcceptHeader, authInfor)
|
||||
|
||||
return httpStatusCode, err
|
||||
}
|
||||
|
||||
func (a testapi) GCScheduleGet(authInfo usrInfo) (int, api_models.AdminJobSchedule, error) {
|
||||
_sling := sling.New().Get(a.basePath)
|
||||
path := "/api/system/gc/schedule"
|
||||
_sling = _sling.Path(path)
|
||||
httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo)
|
||||
var successPayLoad api_models.AdminJobSchedule
|
||||
if 200 == httpStatusCode && nil == err {
|
||||
err = json.Unmarshal(body, &successPayLoad)
|
||||
}
|
||||
|
||||
return httpStatusCode, successPayLoad, err
|
||||
}
|
||||
|
||||
func (a testapi) AddScanAll(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) {
|
||||
_sling := sling.New().Post(a.basePath)
|
||||
|
||||
|
@ -1,147 +0,0 @@
|
||||
// Copyright 2018 Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/goharbor/harbor/src/core/config"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
common_job "github.com/goharbor/harbor/src/common/job"
|
||||
"github.com/goharbor/harbor/src/core/api/models"
|
||||
)
|
||||
|
||||
// GCAPI handles request of harbor GC...
|
||||
type GCAPI struct {
|
||||
AJAPI
|
||||
}
|
||||
|
||||
// Prepare validates the URL and parms, it needs the system admin permission.
|
||||
func (gc *GCAPI) Prepare() {
|
||||
gc.BaseController.Prepare()
|
||||
if !gc.SecurityCtx.IsAuthenticated() {
|
||||
gc.SendUnAuthorizedError(errors.New("UnAuthorized"))
|
||||
return
|
||||
}
|
||||
if !gc.SecurityCtx.IsSysAdmin() {
|
||||
gc.SendForbiddenError(errors.New(gc.SecurityCtx.GetUsername()))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Post according to the request, it creates a cron schedule or a manual trigger for GC.
|
||||
// create a daily schedule for GC
|
||||
// {
|
||||
// "schedule": {
|
||||
// "type": "Daily",
|
||||
// "cron": "0 0 0 * * *"
|
||||
// },
|
||||
// "parameters": {
|
||||
// "delete_untagged": true
|
||||
// }
|
||||
// }
|
||||
// create a manual trigger for GC
|
||||
// {
|
||||
// "schedule": {
|
||||
// "type": "Manual"
|
||||
// },
|
||||
// "parameters": {
|
||||
// "delete_untagged": true
|
||||
// "read_only": true
|
||||
// }
|
||||
// }
|
||||
func (gc *GCAPI) Post() {
|
||||
parameters := make(map[string]interface{})
|
||||
ajr := models.AdminJobReq{
|
||||
Parameters: parameters,
|
||||
}
|
||||
isValid, err := gc.DecodeJSONReqAndValidate(&ajr)
|
||||
if !isValid {
|
||||
gc.SendBadRequestError(err)
|
||||
return
|
||||
}
|
||||
ajr.Parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
|
||||
// default is the non-blocking GC job.
|
||||
ajr.Name = common_job.ImageGC
|
||||
ajr.Parameters["time_window"] = config.GetGCTimeWindow()
|
||||
// if specify read_only:true, API will submit the readonly GC job, otherwise default is non-blocking GC job.
|
||||
readOnlyParam, exist := ajr.Parameters["read_only"]
|
||||
if exist {
|
||||
if readOnly, ok := readOnlyParam.(bool); ok && readOnly {
|
||||
ajr.Name = common_job.ImageGCReadOnly
|
||||
}
|
||||
}
|
||||
gc.submit(&ajr)
|
||||
gc.Redirect(http.StatusCreated, strconv.FormatInt(ajr.ID, 10))
|
||||
}
|
||||
|
||||
// Put handles GC cron schedule update/delete.
|
||||
// Request: delete the schedule of GC
|
||||
// {
|
||||
// "schedule": {
|
||||
// "type": "None",
|
||||
// "cron": ""
|
||||
// },
|
||||
// "parameters": {
|
||||
// "delete_untagged": true
|
||||
// }
|
||||
// }
|
||||
func (gc *GCAPI) Put() {
|
||||
parameters := make(map[string]interface{})
|
||||
ajr := models.AdminJobReq{
|
||||
Parameters: parameters,
|
||||
}
|
||||
isValid, err := gc.DecodeJSONReqAndValidate(&ajr)
|
||||
if !isValid {
|
||||
gc.SendBadRequestError(err)
|
||||
return
|
||||
}
|
||||
ajr.Name = common_job.ImageGC
|
||||
ajr.Parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
|
||||
ajr.Parameters["time_window"] = config.GetGCTimeWindow()
|
||||
gc.updateSchedule(ajr)
|
||||
}
|
||||
|
||||
// GetGC ...
|
||||
func (gc *GCAPI) GetGC() {
|
||||
id, err := gc.GetInt64FromPath(":id")
|
||||
if err != nil {
|
||||
gc.SendInternalServerError(errors.New("need to specify gc id"))
|
||||
return
|
||||
}
|
||||
gc.get(id)
|
||||
}
|
||||
|
||||
// List returns the top 10 executions of GC which includes manual and cron.
|
||||
func (gc *GCAPI) List() {
|
||||
gc.list(common_job.ImageGC)
|
||||
}
|
||||
|
||||
// Get gets GC schedule ...
|
||||
func (gc *GCAPI) Get() {
|
||||
gc.getSchedule(common_job.ImageGC)
|
||||
}
|
||||
|
||||
// GetLog ...
|
||||
func (gc *GCAPI) GetLog() {
|
||||
id, err := gc.GetInt64FromPath(":id")
|
||||
if err != nil {
|
||||
gc.SendBadRequestError(errors.New("invalid ID"))
|
||||
return
|
||||
}
|
||||
gc.getLog(id)
|
||||
}
|
@ -1,39 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/testing/apitests/apilib"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGCPost(t *testing.T) {
|
||||
|
||||
adminJob001 := apilib.AdminJobReq{
|
||||
Parameters: map[string]interface{}{"delete_untagged": false},
|
||||
}
|
||||
assert := assert.New(t)
|
||||
apiTest := newHarborAPI()
|
||||
|
||||
// case 1: add a new admin job
|
||||
code, err := apiTest.AddGC(*admin, adminJob001)
|
||||
if err != nil {
|
||||
t.Error("Error occurred while add a admin job", err.Error())
|
||||
t.Log(err)
|
||||
} else {
|
||||
assert.Equal(201, code, "Add adminjob status should be 201")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGCGet(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
apiTest := newHarborAPI()
|
||||
|
||||
code, _, err := apiTest.GCScheduleGet(*admin)
|
||||
if err != nil {
|
||||
t.Error("Error occurred while get a admin job", err.Error())
|
||||
t.Log(err)
|
||||
} else {
|
||||
assert.Equal(200, code, "Get adminjob status should be 200")
|
||||
}
|
||||
}
|
@ -93,10 +93,11 @@ func (r *DefaultAPIController) CreateRetention(p *policy.Metadata) (int64, error
|
||||
if p.Trigger.Kind == policy.TriggerKindSchedule {
|
||||
cron, ok := p.Trigger.Settings[policy.TriggerSettingsCron]
|
||||
if ok && len(cron.(string)) > 0 {
|
||||
extras := make(map[string]interface{})
|
||||
if _, err = r.scheduler.Schedule(orm.Context(), schedulerVendorType, id, "", cron.(string), SchedulerCallback, TriggerParam{
|
||||
PolicyID: id,
|
||||
Trigger: ExecutionTriggerSchedule,
|
||||
}); err != nil {
|
||||
}, extras); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
@ -152,10 +153,11 @@ func (r *DefaultAPIController) UpdateRetention(p *policy.Metadata) error {
|
||||
}
|
||||
}
|
||||
if needSch {
|
||||
extras := make(map[string]interface{})
|
||||
_, err := r.scheduler.Schedule(orm.Context(), schedulerVendorType, p.ID, "", p.Trigger.Settings[policy.TriggerSettingsCron].(string), SchedulerCallback, TriggerParam{
|
||||
PolicyID: p.ID,
|
||||
Trigger: ExecutionTriggerSchedule,
|
||||
})
|
||||
}, extras)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -220,7 +220,7 @@ func (s *ControllerTestSuite) TestExecution() {
|
||||
type fakeRetentionScheduler struct {
|
||||
}
|
||||
|
||||
func (f *fakeRetentionScheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}) (int64, error) {
|
||||
func (f *fakeRetentionScheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
|
||||
return 111, nil
|
||||
}
|
||||
|
||||
|
@ -34,6 +34,7 @@ type schedule struct {
|
||||
VendorID int64 `orm:"column(vendor_id)"`
|
||||
CRONType string `orm:"column(cron_type)"`
|
||||
CRON string `orm:"column(cron)"`
|
||||
ExtraAttrs string `orm:"column(extra_attrs)"`
|
||||
CallbackFuncName string `orm:"column(callback_func_name)"`
|
||||
CallbackFuncParam string `orm:"column(callback_func_param)"`
|
||||
CreationTime time.Time `orm:"column(creation_time)"`
|
||||
|
@ -45,6 +45,7 @@ func (d *daoTestSuite) SetupTest() {
|
||||
CRON: "0 * * * * *",
|
||||
CallbackFuncName: "callback_func_01",
|
||||
CallbackFuncParam: "callback_func_params",
|
||||
ExtraAttrs: `{"key":"value"}`,
|
||||
}
|
||||
id, err := d.dao.Create(d.ctx, schedule)
|
||||
d.Require().Nil(err)
|
||||
@ -79,6 +80,7 @@ func (d *daoTestSuite) TestGet() {
|
||||
schedule, err = d.dao.Get(d.ctx, d.id)
|
||||
d.Require().Nil(err)
|
||||
d.Equal(d.id, schedule.ID)
|
||||
d.Equal("{\"key\":\"value\"}", schedule.ExtraAttrs)
|
||||
}
|
||||
|
||||
func (d *daoTestSuite) TestDelete() {
|
||||
@ -93,7 +95,7 @@ func (d *daoTestSuite) TestUpdate() {
|
||||
// not found
|
||||
err := d.dao.Update(d.ctx, &schedule{
|
||||
ID: 10000,
|
||||
})
|
||||
}, "CRON")
|
||||
d.True(errors.IsNotFoundErr(err))
|
||||
|
||||
// pass
|
||||
|
@ -37,14 +37,15 @@ var (
|
||||
|
||||
// Schedule describes the detail information about the created schedule
|
||||
type Schedule struct {
|
||||
ID int64 `json:"id"`
|
||||
VendorType string `json:"vendor_type"`
|
||||
VendorID int64 `json:"vendor_id"`
|
||||
CRONType string `json:"cron_type"`
|
||||
CRON string `json:"cron"`
|
||||
Status string `json:"status"` // status of the underlying task(jobservice job)
|
||||
CreationTime time.Time `json:"creation_time"`
|
||||
UpdateTime time.Time `json:"update_time"`
|
||||
ID int64 `json:"id"`
|
||||
VendorType string `json:"vendor_type"`
|
||||
VendorID int64 `json:"vendor_id"`
|
||||
CRONType string `json:"cron_type"`
|
||||
CRON string `json:"cron"`
|
||||
ExtraAttrs map[string]interface{} `json:"extra_attrs"`
|
||||
Status string `json:"status"` // status of the underlying task(jobservice job)
|
||||
CreationTime time.Time `json:"creation_time"`
|
||||
UpdateTime time.Time `json:"update_time"`
|
||||
// we can extend this model to include more information(e.g. how many times the schedule already
|
||||
// runs; when will the schedule runs next time)
|
||||
}
|
||||
@ -59,7 +60,7 @@ type Scheduler interface {
|
||||
// The "params" is passed to the callback function as encoded json string, so the callback
|
||||
// function must decode it before using
|
||||
Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
|
||||
cron string, callbackFuncName string, params interface{}) (int64, error)
|
||||
cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error)
|
||||
// UnScheduleByID the schedule specified by ID
|
||||
UnScheduleByID(ctx context.Context, id int64) error
|
||||
// UnScheduleByVendor the schedule specified by vendor
|
||||
@ -94,10 +95,10 @@ type scheduler struct {
|
||||
// to out of control from the global transaction, and uses a new transaction that only
|
||||
// covers the logic inside the function
|
||||
func (s *scheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
|
||||
cron string, callbackFuncName string, params interface{}) (int64, error) {
|
||||
cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
|
||||
var scheduleID int64
|
||||
f := func(ctx context.Context) error {
|
||||
id, err := s.schedule(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params)
|
||||
id, err := s.schedule(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -113,7 +114,7 @@ func (s *scheduler) Schedule(ctx context.Context, vendorType string, vendorID in
|
||||
}
|
||||
|
||||
func (s *scheduler) schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
|
||||
cron string, callbackFuncName string, params interface{}) (int64, error) {
|
||||
cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
|
||||
if len(vendorType) == 0 {
|
||||
return 0, fmt.Errorf("empty vendor type")
|
||||
}
|
||||
@ -142,6 +143,13 @@ func (s *scheduler) schedule(ctx context.Context, vendorType string, vendorID in
|
||||
}
|
||||
sched.CallbackFuncParam = string(paramsData)
|
||||
}
|
||||
if extras != nil {
|
||||
extrasData, err := json.Marshal(extras)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
sched.ExtraAttrs = string(extrasData)
|
||||
}
|
||||
// create schedule record
|
||||
// when checkin hook comes, the database record must exist,
|
||||
// so the database record must be created first before submitting job
|
||||
@ -272,6 +280,15 @@ func (s *scheduler) convertSchedule(ctx context.Context, schedule *schedule) (*S
|
||||
CreationTime: schedule.CreationTime,
|
||||
UpdateTime: schedule.UpdateTime,
|
||||
}
|
||||
if len(schedule.ExtraAttrs) > 0 {
|
||||
extras := map[string]interface{}{}
|
||||
if err := json.Unmarshal([]byte(schedule.ExtraAttrs), &extras); err != nil {
|
||||
log.Errorf("failed to unmarshal the extra attributes of schedule %d: %v", schedule.ID, err)
|
||||
return nil, err
|
||||
}
|
||||
schd.ExtraAttrs = extras
|
||||
}
|
||||
|
||||
executions, err := s.execMgr.List(ctx, &q.Query{
|
||||
Keywords: map[string]interface{}{
|
||||
"VendorType": JobNameScheduler,
|
||||
|
@ -52,15 +52,16 @@ func (s *schedulerTestSuite) SetupTest() {
|
||||
|
||||
func (s *schedulerTestSuite) TestSchedule() {
|
||||
// empty vendor type
|
||||
id, err := s.scheduler.Schedule(nil, "", 0, "", "0 * * * * *", "callback", nil)
|
||||
extras := make(map[string]interface{})
|
||||
id, err := s.scheduler.Schedule(nil, "", 0, "", "0 * * * * *", "callback", nil, extras)
|
||||
s.NotNil(err)
|
||||
|
||||
// invalid cron
|
||||
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "", "callback", nil)
|
||||
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "", "callback", nil, extras)
|
||||
s.NotNil(err)
|
||||
|
||||
// callback function not exist
|
||||
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "not-exist", nil)
|
||||
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "not-exist", nil, extras)
|
||||
s.NotNil(err)
|
||||
|
||||
// failed to submit to jobservice
|
||||
@ -73,7 +74,7 @@ func (s *schedulerTestSuite) TestSchedule() {
|
||||
Status: job.ErrorStatus.String(),
|
||||
}, nil)
|
||||
s.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
|
||||
_, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param")
|
||||
_, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param", extras)
|
||||
s.Require().NotNil(err)
|
||||
s.dao.AssertExpectations(s.T())
|
||||
s.execMgr.AssertExpectations(s.T())
|
||||
@ -91,7 +92,7 @@ func (s *schedulerTestSuite) TestSchedule() {
|
||||
ExecutionID: 1,
|
||||
Status: job.SuccessStatus.String(),
|
||||
}, nil)
|
||||
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param")
|
||||
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param", extras)
|
||||
s.Require().Nil(err)
|
||||
s.Equal(int64(1), id)
|
||||
s.dao.AssertExpectations(s.T())
|
||||
|
@ -115,6 +115,22 @@ func (t *Task) GetStringFromExtraAttrs(key string) string {
|
||||
return str
|
||||
}
|
||||
|
||||
// GetBoolFromExtraAttrs returns the bool value specified by key
|
||||
func (t *Task) GetBoolFromExtraAttrs(key string) bool {
|
||||
if len(t.ExtraAttrs) == 0 {
|
||||
return false
|
||||
}
|
||||
rt, exist := t.ExtraAttrs[key]
|
||||
if !exist {
|
||||
return false
|
||||
}
|
||||
b, ok := rt.(bool)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Job is the model represents the requested jobservice job
|
||||
type Job struct {
|
||||
Name string
|
||||
|
@ -18,7 +18,7 @@
|
||||
<clr-dg-cell>{{job.createTime | date:'medium'}}</clr-dg-cell>
|
||||
<clr-dg-cell>{{job.updateTime | date:'medium'}}</clr-dg-cell>
|
||||
<clr-dg-cell>
|
||||
<a *ngIf="job.status.toLowerCase() === 'finished' || job.status.toLowerCase() === 'error'" target="_blank" [href]="getLogLink(job.id)"><clr-icon shape="list"></clr-icon></a>
|
||||
<a *ngIf="job.status.toLowerCase() === 'success' || job.status.toLowerCase() === 'error'" target="_blank" [href]="getLogLink(job.id)"><clr-icon shape="list"></clr-icon></a>
|
||||
</clr-dg-cell>
|
||||
</clr-dg-row>
|
||||
<clr-dg-footer>
|
||||
|
@ -52,7 +52,8 @@ func (c *controller) Create(policy *model.Policy) (int64, error) {
|
||||
return 0, err
|
||||
}
|
||||
if isScheduledTrigger(policy) {
|
||||
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, id, "", policy.Trigger.Settings.Cron, CallbackFuncName, id); err != nil {
|
||||
extras := make(map[string]interface{})
|
||||
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, id, "", policy.Trigger.Settings.Cron, CallbackFuncName, id, extras); err != nil {
|
||||
log.Errorf("failed to schedule the policy %d: %v", id, err)
|
||||
}
|
||||
}
|
||||
@ -84,7 +85,8 @@ func (c *controller) Update(policy *model.Policy) error {
|
||||
}
|
||||
// schedule again if needed
|
||||
if isScheduledTrigger(policy) {
|
||||
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, policy.ID, "", policy.Trigger.Settings.Cron, CallbackFuncName, policy.ID); err != nil {
|
||||
extras := make(map[string]interface{})
|
||||
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, policy.ID, "", policy.Trigger.Settings.Cron, CallbackFuncName, policy.ID, extras); err != nil {
|
||||
return fmt.Errorf("failed to schedule the policy %d: %v", policy.ID, err)
|
||||
}
|
||||
}
|
||||
|
@ -226,7 +226,7 @@ func TestCreate(t *testing.T) {
|
||||
|
||||
// scheduled trigger
|
||||
scheduler.On("Schedule", mock.Anything, mock.Anything,
|
||||
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
_, err = ctl.Create(&model.Policy{
|
||||
Enabled: true,
|
||||
Trigger: &model.Trigger{
|
||||
@ -269,7 +269,7 @@ func TestUpdate(t *testing.T) {
|
||||
|
||||
// the trigger changed
|
||||
scheduler.On("Schedule", mock.Anything, mock.Anything,
|
||||
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||
scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything,
|
||||
mock.Anything).Return(nil)
|
||||
|
||||
|
198
src/server/v2.0/handler/gc.go
Normal file
198
src/server/v2.0/handler/gc.go
Normal file
@ -0,0 +1,198 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/go-openapi/runtime/middleware"
|
||||
"github.com/goharbor/harbor/src/controller/gc"
|
||||
"github.com/goharbor/harbor/src/core/config"
|
||||
"github.com/goharbor/harbor/src/lib/errors"
|
||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
"github.com/goharbor/harbor/src/server/v2.0/handler/model"
|
||||
"github.com/goharbor/harbor/src/server/v2.0/models"
|
||||
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/gc"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type gcAPI struct {
|
||||
BaseAPI
|
||||
gcCtr gc.Controller
|
||||
}
|
||||
|
||||
func newGCAPI() *gcAPI {
|
||||
return &gcAPI{
|
||||
gcCtr: gc.NewController(),
|
||||
}
|
||||
}
|
||||
|
||||
func (g *gcAPI) CreateGCSchedule(ctx context.Context, params operation.CreateGCScheduleParams) middleware.Responder {
|
||||
id, err := g.kick(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters)
|
||||
if err != nil {
|
||||
return g.SendError(ctx, err)
|
||||
}
|
||||
// replace the /api/v2.0/system/gc/schedule/{id} to /api/v2.0/system/gc/{id}
|
||||
lastSlashIndex := strings.LastIndex(params.HTTPRequest.URL.Path, "/")
|
||||
if lastSlashIndex != -1 {
|
||||
location := fmt.Sprintf("%s/%d", params.HTTPRequest.URL.Path[:lastSlashIndex], id)
|
||||
return operation.NewCreateGCScheduleCreated().WithLocation(location)
|
||||
}
|
||||
return operation.NewCreateGCScheduleCreated()
|
||||
}
|
||||
|
||||
func (g *gcAPI) UpdateGCSchedule(ctx context.Context, params operation.UpdateGCScheduleParams) middleware.Responder {
|
||||
_, err := g.kick(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters)
|
||||
if err != nil {
|
||||
return g.SendError(ctx, err)
|
||||
}
|
||||
return operation.NewUpdateGCScheduleOK()
|
||||
}
|
||||
|
||||
func (g *gcAPI) kick(ctx context.Context, scheType string, cron string, parameters map[string]interface{}) (int64, error) {
|
||||
// set the required parameters for GC
|
||||
parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
|
||||
parameters["time_window"] = config.GetGCTimeWindow()
|
||||
|
||||
var err error
|
||||
var id int64
|
||||
switch scheType {
|
||||
case ScheduleManual:
|
||||
policy := gc.Policy{
|
||||
ExtraAttrs: parameters,
|
||||
}
|
||||
if dryRun, ok := parameters["dry_run"].(bool); ok {
|
||||
policy.DryRun = dryRun
|
||||
}
|
||||
if deleteUntagged, ok := parameters["delete_untagged"].(bool); ok {
|
||||
policy.DeleteUntagged = deleteUntagged
|
||||
}
|
||||
id, err = g.gcCtr.Start(ctx, policy, task.ExecutionTriggerManual)
|
||||
case ScheduleNone:
|
||||
err = g.gcCtr.DeleteSchedule(ctx)
|
||||
case ScheduleHourly, ScheduleDaily, ScheduleWeekly, ScheduleCustom:
|
||||
policy := gc.Policy{
|
||||
ExtraAttrs: parameters,
|
||||
}
|
||||
if dryRun, ok := parameters["dry_run"].(bool); ok {
|
||||
policy.DryRun = dryRun
|
||||
}
|
||||
if deleteUntagged, ok := parameters["delete_untagged"].(bool); ok {
|
||||
policy.DeleteUntagged = deleteUntagged
|
||||
}
|
||||
err = g.updateSchedule(ctx, scheType, cron, policy)
|
||||
}
|
||||
return id, err
|
||||
}
|
||||
|
||||
func (g *gcAPI) createSchedule(ctx context.Context, cronType, cron string, policy gc.Policy) error {
|
||||
if cron == "" {
|
||||
return errors.New(nil).WithCode(errors.BadRequestCode).
|
||||
WithMessage("empty cron string for gc schedule")
|
||||
}
|
||||
_, err := g.gcCtr.CreateSchedule(ctx, cronType, cron, policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *gcAPI) updateSchedule(ctx context.Context, cronType, cron string, policy gc.Policy) error {
|
||||
if err := g.gcCtr.DeleteSchedule(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.createSchedule(ctx, cronType, cron, policy)
|
||||
}
|
||||
|
||||
func (g *gcAPI) GetGCSchedule(ctx context.Context, params operation.GetGCScheduleParams) middleware.Responder {
|
||||
schedule, err := g.gcCtr.GetSchedule(ctx)
|
||||
if errors.IsNotFoundErr(err) {
|
||||
return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(&scheduler.Schedule{}).ToSwagger())
|
||||
}
|
||||
if err != nil {
|
||||
return g.SendError(ctx, err)
|
||||
}
|
||||
|
||||
return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(schedule).ToSwagger())
|
||||
}
|
||||
|
||||
func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryParams) middleware.Responder {
|
||||
query, err := g.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
|
||||
if err != nil {
|
||||
return g.SendError(ctx, err)
|
||||
}
|
||||
total, err := g.gcCtr.ExecutionCount(ctx, query)
|
||||
if err != nil {
|
||||
return g.SendError(ctx, err)
|
||||
}
|
||||
execs, err := g.gcCtr.ListExecutions(ctx, query)
|
||||
if err != nil {
|
||||
return g.SendError(ctx, err)
|
||||
}
|
||||
|
||||
var hs []*model.GCHistory
|
||||
for _, exec := range execs {
|
||||
extraAttrsString, err := json.Marshal(exec.ExtraAttrs)
|
||||
if err != nil {
|
||||
return g.SendError(ctx, err)
|
||||
}
|
||||
hs = append(hs, &model.GCHistory{
|
||||
ID: exec.ID,
|
||||
Name: gc.GCVendorType,
|
||||
Kind: exec.Trigger,
|
||||
Parameters: string(extraAttrsString),
|
||||
Schedule: &model.ScheduleParam{
|
||||
Type: exec.Trigger,
|
||||
},
|
||||
Status: exec.Status,
|
||||
CreationTime: exec.StartTime,
|
||||
UpdateTime: exec.EndTime,
|
||||
})
|
||||
}
|
||||
|
||||
var results []*models.GCHistory
|
||||
for _, h := range hs {
|
||||
results = append(results, h.ToSwagger())
|
||||
}
|
||||
|
||||
return operation.NewGetGCHistoryOK().
|
||||
WithXTotalCount(total).
|
||||
WithLink(g.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
|
||||
WithPayload(results)
|
||||
}
|
||||
|
||||
func (g *gcAPI) GetGC(ctx context.Context, params operation.GetGCParams) middleware.Responder {
|
||||
exec, err := g.gcCtr.GetExecution(ctx, params.GcID)
|
||||
if err != nil {
|
||||
return g.SendError(ctx, err)
|
||||
}
|
||||
|
||||
extraAttrsString, err := json.Marshal(exec.ExtraAttrs)
|
||||
if err != nil {
|
||||
return g.SendError(ctx, err)
|
||||
}
|
||||
|
||||
res := &model.GCHistory{
|
||||
ID: exec.ID,
|
||||
Name: gc.GCVendorType,
|
||||
Kind: exec.Trigger,
|
||||
Parameters: string(extraAttrsString),
|
||||
Status: exec.Status,
|
||||
Schedule: &model.ScheduleParam{
|
||||
Type: exec.Trigger,
|
||||
},
|
||||
CreationTime: exec.StartTime,
|
||||
UpdateTime: exec.EndTime,
|
||||
}
|
||||
|
||||
return operation.NewGetGCOK().WithPayload(res.ToSwagger())
|
||||
}
|
||||
|
||||
func (g *gcAPI) GetGCLog(ctx context.Context, params operation.GetGCLogParams) middleware.Responder {
|
||||
log, err := g.gcCtr.GetTaskLog(ctx, params.GcID)
|
||||
if err != nil {
|
||||
return g.SendError(ctx, err)
|
||||
}
|
||||
return operation.NewGetGCLogOK().WithPayload(string(log))
|
||||
}
|
@ -41,6 +41,7 @@ func New() http.Handler {
|
||||
ReplicationAPI: newReplicationAPI(),
|
||||
SysteminfoAPI: newSystemInfoAPI(),
|
||||
PingAPI: newPingAPI(),
|
||||
GcAPI: newGCAPI(),
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
|
84
src/server/v2.0/handler/model/gc.go
Normal file
84
src/server/v2.0/handler/model/gc.go
Normal file
@ -0,0 +1,84 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/go-openapi/strfmt"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||
"github.com/goharbor/harbor/src/server/v2.0/models"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ScheduleParam defines the parameter of schedule trigger
|
||||
type ScheduleParam struct {
|
||||
// Daily, Weekly, Custom, Manual, None
|
||||
Type string `json:"type"`
|
||||
// The cron string of scheduled job
|
||||
Cron string `json:"cron"`
|
||||
}
|
||||
|
||||
// GCHistory gc execution history
|
||||
type GCHistory struct {
|
||||
Schedule *ScheduleParam `json:"schedule"`
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"job_name"`
|
||||
Kind string `json:"job_kind"`
|
||||
Parameters string `json:"job_parameters"`
|
||||
Status string `json:"job_status"`
|
||||
UUID string `json:"-"`
|
||||
Deleted bool `json:"deleted"`
|
||||
CreationTime time.Time `json:"creation_time"`
|
||||
UpdateTime time.Time `json:"update_time"`
|
||||
}
|
||||
|
||||
// ToSwagger converts the history to the swagger model
|
||||
func (h *GCHistory) ToSwagger() *models.GCHistory {
|
||||
return &models.GCHistory{
|
||||
ID: h.ID,
|
||||
JobName: h.Name,
|
||||
JobKind: h.Kind,
|
||||
JobParameters: h.Parameters,
|
||||
Deleted: h.Deleted,
|
||||
JobStatus: h.Status,
|
||||
Schedule: &models.ScheduleObj{
|
||||
Cron: h.Schedule.Cron,
|
||||
Type: h.Schedule.Type,
|
||||
},
|
||||
CreationTime: strfmt.DateTime(h.CreationTime),
|
||||
UpdateTime: strfmt.DateTime(h.UpdateTime),
|
||||
}
|
||||
}
|
||||
|
||||
// Schedule ...
|
||||
type Schedule struct {
|
||||
*scheduler.Schedule
|
||||
}
|
||||
|
||||
// ToSwagger converts the schedule to the swagger model
|
||||
// TODO remove the hard code when after issue https://github.com/goharbor/harbor/issues/13047 is resolved.
|
||||
func (s *Schedule) ToSwagger() *models.GCHistory {
|
||||
e, err := json.Marshal(s.ExtraAttrs)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
|
||||
return &models.GCHistory{
|
||||
ID: 0,
|
||||
JobName: "",
|
||||
JobKind: s.CRON,
|
||||
JobParameters: string(e),
|
||||
Deleted: false,
|
||||
JobStatus: "",
|
||||
Schedule: &models.ScheduleObj{
|
||||
Cron: s.CRON,
|
||||
Type: "Custom",
|
||||
},
|
||||
CreationTime: strfmt.DateTime(s.CreationTime),
|
||||
UpdateTime: strfmt.DateTime(s.UpdateTime),
|
||||
}
|
||||
}
|
||||
|
||||
// NewSchedule ...
|
||||
func NewSchedule(s *scheduler.Schedule) *Schedule {
|
||||
return &Schedule{Schedule: s}
|
||||
}
|
@ -29,6 +29,21 @@ import (
|
||||
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
// ScheduleHourly : 'Hourly'
|
||||
ScheduleHourly = "Hourly"
|
||||
// ScheduleDaily : 'Daily'
|
||||
ScheduleDaily = "Daily"
|
||||
// ScheduleWeekly : 'Weekly'
|
||||
ScheduleWeekly = "Weekly"
|
||||
// ScheduleCustom : 'Custom'
|
||||
ScheduleCustom = "Custom"
|
||||
// ScheduleManual : 'Manual'
|
||||
ScheduleManual = "Manual"
|
||||
// ScheduleNone : 'None'
|
||||
ScheduleNone = "None"
|
||||
)
|
||||
|
||||
func boolValue(v *bool) bool {
|
||||
if v != nil {
|
||||
return *v
|
||||
|
@ -45,10 +45,6 @@ func registerLegacyRoutes() {
|
||||
beego.Router("/api/"+version+"/quotas", &api.QuotaAPI{}, "get:List")
|
||||
beego.Router("/api/"+version+"/quotas/:id([0-9]+)", &api.QuotaAPI{}, "get:Get;put:Put")
|
||||
|
||||
beego.Router("/api/"+version+"/system/gc", &api.GCAPI{}, "get:List")
|
||||
beego.Router("/api/"+version+"/system/gc/:id", &api.GCAPI{}, "get:GetGC")
|
||||
beego.Router("/api/"+version+"/system/gc/:id([0-9]+)/log", &api.GCAPI{}, "get:GetLog")
|
||||
beego.Router("/api/"+version+"/system/gc/schedule", &api.GCAPI{}, "get:Get;put:Put;post:Post")
|
||||
beego.Router("/api/"+version+"/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post")
|
||||
beego.Router("/api/"+version+"/system/CVEAllowlist", &api.SysCVEAllowlistAPI{}, "get:Get;put:Put")
|
||||
beego.Router("/api/"+version+"/system/oidc/ping", &api.OIDCAPI{}, "post:Ping")
|
||||
|
@ -62,20 +62,20 @@ func (_m *Scheduler) ListSchedules(ctx context.Context, query *q.Query) ([]*sche
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// Schedule provides a mock function with given fields: ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params
|
||||
func (_m *Scheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}) (int64, error) {
|
||||
ret := _m.Called(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params)
|
||||
// Schedule provides a mock function with given fields: ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras
|
||||
func (_m *Scheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
|
||||
ret := _m.Called(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
|
||||
|
||||
var r0 int64
|
||||
if rf, ok := ret.Get(0).(func(context.Context, string, int64, string, string, string, interface{}) int64); ok {
|
||||
r0 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params)
|
||||
if rf, ok := ret.Get(0).(func(context.Context, string, int64, string, string, string, interface{}, map[string]interface{}) int64); ok {
|
||||
r0 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func(context.Context, string, int64, string, string, string, interface{}) error); ok {
|
||||
r1 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params)
|
||||
if rf, ok := ret.Get(1).(func(context.Context, string, int64, string, string, string, interface{}, map[string]interface{}) error); ok {
|
||||
r1 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
@ -61,6 +61,7 @@ def _create_client(server, credential, debug, api_type="products"):
|
||||
"scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)),
|
||||
"replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)),
|
||||
"robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)),
|
||||
"gc": v2_swagger_client.GcApi(v2_swagger_client.ApiClient(cfg)),
|
||||
}.get(api_type,'Error: Wrong API type')
|
||||
|
||||
def _assert_status_code(expect_code, return_code):
|
||||
|
134
tests/apitests/python/library/gc.py
Normal file
134
tests/apitests/python/library/gc.py
Normal file
@ -0,0 +1,134 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import time
|
||||
import base
|
||||
import re
|
||||
import v2_swagger_client
|
||||
from v2_swagger_client.rest import ApiException
|
||||
|
||||
class GC(base.Base, object):
|
||||
def __init__(self):
|
||||
super(GC,self).__init__(api_type = "gc")
|
||||
|
||||
def get_gc_history(self, expect_status_code = 200, expect_response_body = None, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
|
||||
try:
|
||||
data, status_code, _ = client.get_gc_history_with_http_info()
|
||||
except ApiException as e:
|
||||
if e.status == expect_status_code:
|
||||
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
|
||||
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
|
||||
else:
|
||||
return e.reason, e.body
|
||||
else:
|
||||
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
|
||||
base._assert_status_code(expect_status_code, status_code)
|
||||
return data
|
||||
|
||||
def get_gc_status_by_id(self, job_id, expect_status_code = 200, expect_response_body = None, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
|
||||
try:
|
||||
data, status_code, _ = client.get_gc_with_http_info(job_id)
|
||||
except ApiException as e:
|
||||
if e.status == expect_status_code:
|
||||
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
|
||||
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
|
||||
else:
|
||||
return e.reason, e.body
|
||||
else:
|
||||
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
|
||||
base._assert_status_code(expect_status_code, status_code)
|
||||
return data
|
||||
|
||||
def get_gc_log_by_id(self, job_id, expect_status_code = 200, expect_response_body = None, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
|
||||
try:
|
||||
data, status_code, _ = client.get_gc_log_with_http_info(job_id)
|
||||
except ApiException as e:
|
||||
if e.status == expect_status_code:
|
||||
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
|
||||
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
|
||||
else:
|
||||
return e.reason, e.body
|
||||
else:
|
||||
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
|
||||
base._assert_status_code(expect_status_code, status_code)
|
||||
return data
|
||||
|
||||
def get_gc_schedule(self, expect_status_code = 200, expect_response_body = None, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
|
||||
try:
|
||||
data, status_code, _ = client.get_gc_schedule_with_http_info()
|
||||
except ApiException as e:
|
||||
if e.status == expect_status_code:
|
||||
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
|
||||
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
|
||||
else:
|
||||
return e.reason, e.body
|
||||
else:
|
||||
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
|
||||
base._assert_status_code(expect_status_code, status_code)
|
||||
return data
|
||||
|
||||
def create_gc_schedule(self, schedule_type, is_delete_untagged, cron = None, expect_status_code = 201, expect_response_body = None, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
|
||||
gc_parameters = {'delete_untagged':is_delete_untagged}
|
||||
|
||||
gc_schedule = v2_swagger_client.ScheduleObj()
|
||||
gc_schedule.type = schedule_type
|
||||
if cron is not None:
|
||||
gc_schedule.cron = cron
|
||||
|
||||
gc_job = v2_swagger_client.Schedule()
|
||||
gc_job.schedule = gc_schedule
|
||||
gc_job.parameters = gc_parameters
|
||||
|
||||
try:
|
||||
_, status_code, header = client.create_gc_schedule_with_http_info(gc_job)
|
||||
except ApiException as e:
|
||||
if e.status == expect_status_code:
|
||||
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
|
||||
raise Exception(r"Create GC schedule response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
|
||||
else:
|
||||
return e.reason, e.body
|
||||
else:
|
||||
raise Exception(r"Create GC schedule result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
|
||||
base._assert_status_code(expect_status_code, status_code)
|
||||
return base._get_id_from_header(header)
|
||||
|
||||
def gc_now(self, is_delete_untagged=False, **kwargs):
|
||||
gc_id = self.create_gc_schedule('Manual', is_delete_untagged, **kwargs)
|
||||
return gc_id
|
||||
|
||||
def validate_gc_job_status(self, gc_id, expected_gc_status, **kwargs):
|
||||
get_gc_status_finish = False
|
||||
timeout_count = 20
|
||||
while timeout_count > 0:
|
||||
time.sleep(5)
|
||||
status = self.get_gc_status_by_id(gc_id, **kwargs)
|
||||
print("GC job No: {}, status: {}".format(timeout_count, status.job_status))
|
||||
if status.job_status == expected_gc_status:
|
||||
get_gc_status_finish = True
|
||||
break
|
||||
timeout_count = timeout_count - 1
|
||||
|
||||
if not (get_gc_status_finish):
|
||||
raise Exception("GC status is not as expected '{}' actual GC status is '{}'".format(expected_gc_status, status.job_status))
|
||||
|
||||
def validate_deletion_success(self, gc_id, **kwargs):
|
||||
log_content = self.get_gc_log_by_id(gc_id, **kwargs)
|
||||
key_message = "manifests eligible for deletion"
|
||||
key_message_pos = log_content.find(key_message)
|
||||
full_message = log_content[key_message_pos-30 : key_message_pos + len(key_message)]
|
||||
deleted_files_count_list = re.findall(r'\s+(\d+)\s+blobs\s+and\s+\d+\s+manifests\s+eligible\s+for\s+deletion', full_message)
|
||||
|
||||
if len(deleted_files_count_list) != 1:
|
||||
raise Exception(r"Fail to get blobs eligible for deletion in log file, failure is {}.".format(len(deleted_files_count_list)))
|
||||
deleted_files_count = int(deleted_files_count_list[0])
|
||||
if deleted_files_count == 0:
|
||||
raise Exception(r"Get blobs eligible for deletion count is {}, while we expect more than 1.".format(deleted_files_count))
|
@ -157,38 +157,6 @@ class System(base.Base):
|
||||
scan_all_id = self.create_scan_all_schedule('Manual', **kwargs)
|
||||
return scan_all_id
|
||||
|
||||
def gc_now(self, is_delete_untagged=False, **kwargs):
|
||||
gc_id = self.create_gc_schedule('Manual', is_delete_untagged, **kwargs)
|
||||
return gc_id
|
||||
|
||||
def validate_gc_job_status(self, gc_id, expected_gc_status, **kwargs):
|
||||
get_gc_status_finish = False
|
||||
timeout_count = 20
|
||||
while timeout_count > 0:
|
||||
time.sleep(5)
|
||||
status = self.get_gc_status_by_id(gc_id, **kwargs)
|
||||
print("GC job No: {}, status: {}".format(timeout_count, status.job_status))
|
||||
if status.job_status == expected_gc_status:
|
||||
get_gc_status_finish = True
|
||||
break
|
||||
timeout_count = timeout_count - 1
|
||||
|
||||
if not (get_gc_status_finish):
|
||||
raise Exception("GC status is not as expected '{}' actual GC status is '{}'".format(expected_gc_status, status.job_status))
|
||||
|
||||
def validate_deletion_success(self, gc_id, **kwargs):
|
||||
log_content = self.get_gc_log_by_id(gc_id, **kwargs)
|
||||
key_message = "manifests eligible for deletion"
|
||||
key_message_pos = log_content.find(key_message)
|
||||
full_message = log_content[key_message_pos-30 : key_message_pos + len(key_message)]
|
||||
deleted_files_count_list = re.findall(r'\s+(\d+)\s+blobs\s+and\s+\d+\s+manifests\s+eligible\s+for\s+deletion', full_message)
|
||||
|
||||
if len(deleted_files_count_list) != 1:
|
||||
raise Exception(r"Fail to get blobs eligible for deletion in log file, failure is {}.".format(len(deleted_files_count_list)))
|
||||
deleted_files_count = int(deleted_files_count_list[0])
|
||||
if deleted_files_count == 0:
|
||||
raise Exception(r"Get blobs eligible for deletion count is {}, while we expect more than 1.".format(deleted_files_count))
|
||||
|
||||
def set_cve_allowlist(self, expires_at=None, expected_status_code=200, *cve_ids, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
cve_list = [swagger_client.CVEAllowlistItem(cve_id=c) for c in cve_ids]
|
||||
|
@ -7,17 +7,17 @@ from testutils import ADMIN_CLIENT, suppress_urllib3_warning
|
||||
from testutils import TEARDOWN
|
||||
from testutils import harbor_server
|
||||
from library.user import User
|
||||
from library.system import System
|
||||
from library.project import Project
|
||||
from library.repository import Repository
|
||||
from library.base import _assert_status_code
|
||||
from library.repository import push_special_image_to_project
|
||||
from library.artifact import Artifact
|
||||
from library.gc import GC
|
||||
|
||||
class TestProjects(unittest.TestCase):
|
||||
@suppress_urllib3_warning
|
||||
def setUp(self):
|
||||
self.system = System()
|
||||
self.gc = GC()
|
||||
self.project = Project()
|
||||
self.user = User()
|
||||
self.repo = Repository()
|
||||
@ -82,13 +82,13 @@ class TestProjects(unittest.TestCase):
|
||||
self.artifact.delete_tag(TestProjects.project_gc_untag_name, self.repo_name_untag, self.tag, self.tag, **ADMIN_CLIENT)
|
||||
|
||||
#5. Tigger garbage collection operation;
|
||||
gc_id = self.system.gc_now(**ADMIN_CLIENT)
|
||||
gc_id = self.gc.gc_now(**ADMIN_CLIENT)
|
||||
|
||||
#6. Check garbage collection job was finished;
|
||||
self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT)
|
||||
self.gc.validate_gc_job_status(gc_id, "Success", **ADMIN_CLIENT)
|
||||
|
||||
#7. Get garbage collection log, check there is a number of files was deleted;
|
||||
self.system.validate_deletion_success(gc_id, **ADMIN_CLIENT)
|
||||
self.gc.validate_deletion_success(gc_id, **ADMIN_CLIENT)
|
||||
|
||||
artifacts = self.artifact.list_artifacts(TestProjects.project_gc_untag_name, self.repo_name_untag, **TestProjects.USER_GC_CLIENT)
|
||||
_assert_status_code(len(artifacts), 1)
|
||||
@ -96,13 +96,13 @@ class TestProjects(unittest.TestCase):
|
||||
time.sleep(5)
|
||||
|
||||
#9. Tigger garbage collection operation;
|
||||
gc_id = self.system.gc_now(is_delete_untagged=True, **ADMIN_CLIENT)
|
||||
gc_id = self.gc.gc_now(is_delete_untagged=True, **ADMIN_CLIENT)
|
||||
|
||||
#10. Check garbage collection job was finished;
|
||||
self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT)
|
||||
self.gc.validate_gc_job_status(gc_id, "Success", **ADMIN_CLIENT)
|
||||
|
||||
#7. Get garbage collection log, check there is a number of files was deleted;
|
||||
self.system.validate_deletion_success(gc_id, **ADMIN_CLIENT)
|
||||
self.gc.validate_deletion_success(gc_id, **ADMIN_CLIENT)
|
||||
|
||||
#11. Repository with untag image should be still there;
|
||||
repo_data_untag = self.repo.list_repositories(TestProjects.project_gc_untag_name, **TestProjects.USER_GC_CLIENT)
|
||||
|
Loading…
Reference in New Issue
Block a user