Merge pull request #13139 from wy65701436/migrate-gc

Migrate gc to task manager
This commit is contained in:
Wenkai Yin(尹文开) 2020-12-14 10:43:44 +08:00 committed by GitHub
commit 6569016d35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 1179 additions and 414 deletions

View File

@ -1549,25 +1549,6 @@ paths:
description: Only admin has this authority. description: Only admin has this authority.
'500': '500':
description: Unexpected internal errors. description: Unexpected internal errors.
/system/gc:
get:
summary: Get gc results.
description: This endpoint let user get latest ten gc results.
tags:
- Products
responses:
'200':
description: Get gc results successfully.
schema:
type: array
items:
$ref: '#/definitions/GCResult'
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'500':
description: Unexpected internal errors.
'/system/gc/{id}': '/system/gc/{id}':
get: get:
summary: Get gc status. summary: Get gc status.
@ -1592,101 +1573,6 @@ paths:
description: User does not have permission of admin role. description: User does not have permission of admin role.
'500': '500':
description: Unexpected internal errors. description: Unexpected internal errors.
'/system/gc/{id}/log':
get:
summary: Get gc job log.
description: This endpoint let user get gc job logs filtered by specific ID.
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Relevant job ID
tags:
- Products
responses:
'200':
description: Get successfully.
schema:
type: string
'400':
description: Illegal format of provided ID value.
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'404':
description: The specific gc ID's log does not exist.
'500':
description: Unexpected internal errors.
/system/gc/schedule:
get:
summary: Get gc's schedule.
description: This endpoint is for get schedule of gc job.
tags:
- Products
responses:
'200':
description: Get gc's schedule.
schema:
$ref: '#/definitions/AdminJobSchedule'
'401':
description: User need to log in first.
'403':
description: Only admin has this authority.
'500':
description: Unexpected internal errors.
put:
summary: Update gc's schedule.
description: |
This endpoint is for update gc schedule.
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/AdminJobSchedule'
description: Updates of gc's schedule.
tags:
- Products
responses:
'200':
description: Updated gc's schedule successfully.
'400':
description: Invalid schedule type.
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'500':
description: Unexpected internal errors.
post:
summary: Create a gc schedule.
description: |
This endpoint is for update gc schedule.
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/AdminJobSchedule'
description: Updates of gc's schedule.
tags:
- Products
responses:
'200':
description: GC schedule successfully.
'400':
description: Invalid schedule type.
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'409':
description: There is a "gc" job in progress, so the request cannot be served.
'500':
description: Unexpected internal errors.
/system/scanAll/schedule: /system/scanAll/schedule:
get: get:
summary: Get scan_all's schedule. summary: Get scan_all's schedule.

View File

@ -2017,6 +2017,155 @@ paths:
description: Not found the default root certificate. description: Not found the default root certificate.
'500': '500':
$ref: '#/responses/500' $ref: '#/responses/500'
/system/gc:
get:
summary: Get gc results.
description: This endpoint let user get gc execution history.
tags:
- gc
operationId: getGCHistory
parameters:
- $ref: '#/parameters/query'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
responses:
'200':
description: Get gc results successfully.
headers:
X-Total-Count:
description: The total count of history
type: integer
Link:
description: Link refers to the previous page and next page
type: string
schema:
type: array
items:
$ref: '#/definitions/GCHistory'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
/system/gc/{gc_id}:
get:
summary: Get gc status.
description: This endpoint let user get gc status filtered by specific ID.
operationId: getGC
parameters:
- $ref: '#/parameters/gcId'
tags:
- gc
responses:
'200':
description: Get gc results successfully.
schema:
$ref: '#/definitions/GCHistory'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
/system/gc/{gc_id}/log:
get:
summary: Get gc job log.
description: This endpoint let user get gc job logs filtered by specific ID.
operationId: getGCLog
parameters:
- $ref: '#/parameters/gcId'
tags:
- gc
produces:
- text/plain
responses:
'200':
description: Get successfully.
schema:
type: string
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
/system/gc/schedule:
get:
summary: Get gc's schedule.
description: This endpoint is for get schedule of gc job.
operationId: getGCSchedule
tags:
- gc
responses:
'200':
description: Get gc's schedule.
schema:
$ref: '#/definitions/GCHistory'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
post:
summary: Create a gc schedule.
description: |
This endpoint is for update gc schedule.
operationId: createGCSchedule
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/Schedule'
description: Updates of gc's schedule.
tags:
- gc
responses:
'201':
$ref: '#/responses/201'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'409':
$ref: '#/responses/409'
'500':
$ref: '#/responses/500'
put:
summary: Update gc's schedule.
description: |
This endpoint is for update gc schedule.
operationId: updateGCSchedule
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/Schedule'
description: Updates of gc's schedule.
tags:
- gc
responses:
'200':
description: Updated gc's schedule successfully.
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
/ping: /ping:
get: get:
summary: Ping Harbor to check if it's alive. summary: Ping Harbor to check if it's alive.
@ -2134,6 +2283,13 @@ parameters:
description: Robot ID description: Robot ID
required: true required: true
type: integer type: integer
gcId:
name: gc_id
in: path
description: The ID of the gc log
required: true
type: integer
format: int64
responses: responses:
'200': '200':
description: Success description: Success
@ -3351,3 +3507,55 @@ definitions:
description: The storage of system. description: The storage of system.
items: items:
$ref: '#/definitions/Storage' $ref: '#/definitions/Storage'
GCHistory:
type: object
properties:
id:
type: integer
description: the id of gc job.
job_name:
type: string
description: the job name of gc job.
job_kind:
type: string
description: the job kind of gc job.
job_parameters:
type: string
description: the job parameters of gc job.
schedule:
$ref: '#/definitions/ScheduleObj'
job_status:
type: string
description: the status of gc job.
deleted:
type: boolean
description: if gc job was deleted.
creation_time:
type: string
format: date-time
description: the creation time of gc job.
update_time:
type: string
format: date-time
description: the update time of gc job.
Schedule:
type: object
properties:
schedule:
$ref: '#/definitions/ScheduleObj'
parameters:
type: object
description: The parameters of admin job
additionalProperties:
type: object
ScheduleObj:
type: object
properties:
type:
type: string
description: |
The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manually' and 'None'.
'Manually' means to trigger it right away and 'None' means to cancel the schedule.
cron:
type: string
description: A cron expression, a time-based job scheduler.

View File

@ -1,4 +1,4 @@
/* /*
Fixes issue https://github.com/goharbor/harbor/issues/13317 Fixes issue https://github.com/goharbor/harbor/issues/13317
Ensure the role_id of maintainer is 4 and the role_id of limisted guest is 5 Ensure the role_id of maintainer is 4 and the role_id of limisted guest is 5
*/ */
@ -268,3 +268,7 @@ BEGIN
UPDATE scanner_registration SET is_default = TRUE WHERE name = 'Trivy' AND immutable = TRUE; UPDATE scanner_registration SET is_default = TRUE WHERE name = 'Trivy' AND immutable = TRUE;
END IF; END IF;
END $$; END $$;
ALTER TABLE execution ALTER COLUMN vendor_type type varchar(64);
ALTER TABLE schedule ALTER COLUMN vendor_type type varchar(64);
ALTER TABLE schedule ADD COLUMN IF NOT EXISTS extra_attrs JSON;
ALTER TABLE task ALTER COLUMN vendor_type type varchar(64);

View File

@ -0,0 +1,27 @@
package gc
import (
"context"
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
)
func init() {
err := scheduler.RegisterCallbackFunc(SchedulerCallback, gcCallback)
if err != nil {
log.Fatalf("failed to registry GC call back, %v", err)
}
}
func gcCallback(ctx context.Context, p string) error {
param := &Policy{}
if err := json.Unmarshal([]byte(p), param); err != nil {
return fmt.Errorf("failed to unmarshal the param: %v", err)
}
_, err := Ctl.Start(orm.Context(), *param, task.ExecutionTriggerSchedule)
return err
}

View File

@ -0,0 +1,217 @@
package gc
import (
"context"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
)
var (
// Ctl is a global garbage collection controller instance
Ctl = NewController()
)
const (
// SchedulerCallback ...
SchedulerCallback = "GARBAGE_COLLECTION"
// GCVendorType ...
GCVendorType = "GARBAGE_COLLECTION"
)
// Controller manages the tags
type Controller interface {
// Start start a manual gc job
Start(ctx context.Context, policy Policy, trigger string) (int64, error)
// Stop stop a gc job
Stop(ctx context.Context, id int64) error
// ExecutionCount returns the total count of executions according to the query
ExecutionCount(ctx context.Context, query *q.Query) (count int64, err error)
// ListExecutions lists the executions according to the query
ListExecutions(ctx context.Context, query *q.Query) (executions []*Execution, err error)
// GetExecution gets the specific execution
GetExecution(ctx context.Context, executionID int64) (execution *Execution, err error)
// GetTask gets the specific task
GetTask(ctx context.Context, id int64) (*Task, error)
// GetTaskLog gets log of the specific task
GetTaskLog(ctx context.Context, id int64) ([]byte, error)
// GetSchedule get the current gc schedule
GetSchedule(ctx context.Context) (*scheduler.Schedule, error)
// CreateSchedule create the gc schedule with cron type & string
CreateSchedule(ctx context.Context, cronType, cron string, policy Policy) (int64, error)
// DeleteSchedule remove the gc schedule
DeleteSchedule(ctx context.Context) error
}
// NewController creates an instance of the default repository controller
func NewController() Controller {
return &controller{
taskMgr: task.NewManager(),
exeMgr: task.NewExecutionManager(),
schedulerMgr: scheduler.New(),
}
}
type controller struct {
taskMgr task.Manager
exeMgr task.ExecutionManager
schedulerMgr scheduler.Scheduler
}
// Start starts the manual GC
func (c *controller) Start(ctx context.Context, policy Policy, trigger string) (int64, error) {
para := make(map[string]interface{})
para["delete_untagged"] = policy.DeleteUntagged
para["dry_run"] = policy.DryRun
para["redis_url_reg"] = policy.ExtraAttrs["redis_url_reg"]
para["time_window"] = policy.ExtraAttrs["time_window"]
execID, err := c.exeMgr.Create(ctx, GCVendorType, -1, trigger, para)
if err != nil {
return -1, err
}
_, err = c.taskMgr.Create(ctx, execID, &task.Job{
Name: job.ImageGC,
Metadata: &job.Metadata{
JobKind: job.KindGeneric,
},
Parameters: para,
})
if err != nil {
return -1, err
}
return execID, nil
}
// Stop ...
func (c *controller) Stop(ctx context.Context, id int64) error {
return c.exeMgr.Stop(ctx, id)
}
// ExecutionCount ...
func (c *controller) ExecutionCount(ctx context.Context, query *q.Query) (int64, error) {
query.Keywords["VendorType"] = GCVendorType
return c.exeMgr.Count(ctx, query)
}
// ListExecutions ...
func (c *controller) ListExecutions(ctx context.Context, query *q.Query) ([]*Execution, error) {
query = q.MustClone(query)
query.Keywords["VendorType"] = GCVendorType
execs, err := c.exeMgr.List(ctx, query)
if err != nil {
return nil, err
}
var executions []*Execution
for _, exec := range execs {
executions = append(executions, convertExecution(exec))
}
return executions, nil
}
// GetExecution ...
func (c *controller) GetExecution(ctx context.Context, id int64) (*Execution, error) {
execs, err := c.exeMgr.List(ctx, &q.Query{
Keywords: map[string]interface{}{
"ID": id,
"VendorType": GCVendorType,
},
})
if err != nil {
return nil, err
}
if len(execs) == 0 {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).
WithMessage("garbage collection execution %d not found", id)
}
return convertExecution(execs[0]), nil
}
// GetTask ...
func (c *controller) GetTask(ctx context.Context, id int64) (*Task, error) {
tasks, err := c.taskMgr.List(ctx, &q.Query{
Keywords: map[string]interface{}{
"ID": id,
"VendorType": GCVendorType,
},
})
if err != nil {
return nil, err
}
if len(tasks) == 0 {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).
WithMessage("garbage collection task %d not found", id)
}
return convertTask(tasks[0]), nil
}
// GetTaskLog ...
func (c *controller) GetTaskLog(ctx context.Context, id int64) ([]byte, error) {
_, err := c.GetTask(ctx, id)
if err != nil {
return nil, err
}
return c.taskMgr.GetLog(ctx, id)
}
// GetSchedule ...
func (c *controller) GetSchedule(ctx context.Context) (*scheduler.Schedule, error) {
sch, err := c.schedulerMgr.ListSchedules(ctx, q.New(q.KeyWords{"VendorType": GCVendorType}))
if err != nil {
return nil, err
}
if len(sch) == 0 {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc schedule is found")
}
if sch[0] == nil {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc schedule is found")
}
return sch[0], nil
}
// CreateSchedule ...
func (c *controller) CreateSchedule(ctx context.Context, cronType, cron string, policy Policy) (int64, error) {
extras := make(map[string]interface{})
extras["delete_untagged"] = policy.DeleteUntagged
return c.schedulerMgr.Schedule(ctx, GCVendorType, -1, cronType, cron, SchedulerCallback, policy, extras)
}
// DeleteSchedule ...
func (c *controller) DeleteSchedule(ctx context.Context) error {
return c.schedulerMgr.UnScheduleByVendor(ctx, GCVendorType, -1)
}
func convertExecution(exec *task.Execution) *Execution {
return &Execution{
ID: exec.ID,
Status: exec.Status,
StatusMessage: exec.StatusMessage,
Trigger: exec.Trigger,
ExtraAttrs: exec.ExtraAttrs,
StartTime: exec.StartTime,
EndTime: exec.EndTime,
}
}
func convertTask(task *task.Task) *Task {
return &Task{
ID: task.ID,
ExecutionID: task.ExecutionID,
Status: task.Status,
StatusMessage: task.StatusMessage,
RunCount: task.RunCount,
DeleteUntagged: task.GetBoolFromExtraAttrs("delete_untagged"),
DryRun: task.GetBoolFromExtraAttrs("dry_run"),
JobID: task.JobID,
CreationTime: task.CreationTime,
StartTime: task.StartTime,
UpdateTime: task.UpdateTime,
EndTime: task.EndTime,
}
}

View File

@ -0,0 +1,148 @@
package gc
import (
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
"github.com/goharbor/harbor/src/testing/mock"
schedulertesting "github.com/goharbor/harbor/src/testing/pkg/scheduler"
tasktesting "github.com/goharbor/harbor/src/testing/pkg/task"
"github.com/stretchr/testify/suite"
"testing"
)
type gcCtrTestSuite struct {
suite.Suite
scheduler *schedulertesting.Scheduler
execMgr *tasktesting.ExecutionManager
taskMgr *tasktesting.Manager
ctl *controller
}
func (g *gcCtrTestSuite) SetupTest() {
g.execMgr = &tasktesting.ExecutionManager{}
g.taskMgr = &tasktesting.Manager{}
g.scheduler = &schedulertesting.Scheduler{}
g.ctl = &controller{
taskMgr: g.taskMgr,
exeMgr: g.execMgr,
schedulerMgr: g.scheduler,
}
}
func (g *gcCtrTestSuite) TestStart() {
g.execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
g.taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
g.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
dataMap := make(map[string]interface{})
p := Policy{
DeleteUntagged: true,
ExtraAttrs: dataMap,
}
id, err := g.ctl.Start(nil, p, task.ExecutionTriggerManual)
g.Nil(err)
g.Equal(int64(1), id)
}
func (g *gcCtrTestSuite) TestStop() {
g.execMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
g.Nil(g.ctl.Stop(nil, 1))
}
func (g *gcCtrTestSuite) TestGetTaskLog() {
g.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
{
ID: 1,
ExecutionID: 1,
Status: job.SuccessStatus.String(),
},
}, nil)
g.taskMgr.On("GetLog", mock.Anything, mock.Anything).Return([]byte("hello world"), nil)
log, err := g.ctl.GetTaskLog(nil, 1)
g.Nil(err)
g.Equal([]byte("hello world"), log)
}
func (g *gcCtrTestSuite) TestExecutionCount() {
g.execMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil)
count, err := g.ctl.ExecutionCount(nil, q.New(q.KeyWords{"VendorType": "gc"}))
g.Nil(err)
g.Equal(int64(1), count)
}
func (g *gcCtrTestSuite) TestGetExecution() {
g.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
{
ID: 1,
Trigger: "Manual",
VendorType: GCVendorType,
StatusMessage: "Success",
},
}, nil)
hs, err := g.ctl.GetExecution(nil, int64(1))
g.Nil(err)
g.Equal("Manual", hs.Trigger)
}
func (g *gcCtrTestSuite) TestListExecutions() {
g.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
{
ID: 1,
Trigger: "Manual",
},
}, nil)
g.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
{
ID: 112,
ExecutionID: 1,
Status: job.SuccessStatus.String(),
},
}, nil)
hs, err := g.ctl.ListExecutions(nil, q.New(q.KeyWords{"VendorType": "gc"}))
g.Nil(err)
g.Equal("Manual", hs[0].Trigger)
}
func (g *gcCtrTestSuite) TestGetSchedule() {
g.scheduler.On("ListSchedules", mock.Anything, mock.Anything).Return([]*scheduler.Schedule{
{
ID: 1,
VendorType: "gc",
},
}, nil)
sche, err := g.ctl.GetSchedule(nil)
g.Nil(err)
g.Equal("gc", sche.VendorType)
}
func (g *gcCtrTestSuite) TestCreateSchedule() {
g.scheduler.On("Schedule", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
dataMap := make(map[string]interface{})
p := Policy{
DeleteUntagged: true,
ExtraAttrs: dataMap,
}
id, err := g.ctl.CreateSchedule(nil, "Daily", "* * * * * *", p)
g.Nil(err)
g.Equal(int64(1), id)
}
func (g *gcCtrTestSuite) TestDeleteSchedule() {
g.scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything, mock.Anything).Return(nil)
g.Nil(g.ctl.DeleteSchedule(nil))
}
func TestControllerTestSuite(t *testing.T) {
suite.Run(t, &gcCtrTestSuite{})
}

View File

@ -0,0 +1,54 @@
package gc
import (
"time"
)
// Policy ...
type Policy struct {
Trigger *Trigger `json:"trigger"`
DeleteUntagged bool `json:"bool"`
DryRun bool `json:"dryrun"`
ExtraAttrs map[string]interface{} `json:"extra_attrs"`
}
// TriggerType represents the type of trigger.
type TriggerType string
// Trigger holds info for a trigger
type Trigger struct {
Type TriggerType `json:"type"`
Settings *TriggerSettings `json:"trigger_settings"`
}
// TriggerSettings is the setting about the trigger
type TriggerSettings struct {
Cron string `json:"cron"`
}
// Execution model for replication
type Execution struct {
ID int64
Status string
StatusMessage string
Trigger string
ExtraAttrs map[string]interface{}
StartTime time.Time
EndTime time.Time
}
// Task model for replication
type Task struct {
ID int64
ExecutionID int64
Status string
StatusMessage string
RunCount int32
DeleteUntagged bool
DryRun bool
JobID string
CreationTime time.Time
StartTime time.Time
UpdateTime time.Time
EndTime time.Time
}

View File

@ -292,8 +292,9 @@ func (c *controller) CreatePolicy(ctx context.Context, schema *policyModels.Sche
schema.Trigger.Type == policyModels.TriggerTypeScheduled && schema.Trigger.Type == policyModels.TriggerTypeScheduled &&
len(schema.Trigger.Settings.Cron) > 0 { len(schema.Trigger.Settings.Cron) > 0 {
// schedule and update policy // schedule and update policy
extras := make(map[string]interface{})
if _, err = c.scheduler.Schedule(ctx, job.P2PPreheat, id, "", schema.Trigger.Settings.Cron, if _, err = c.scheduler.Schedule(ctx, job.P2PPreheat, id, "", schema.Trigger.Settings.Cron,
SchedulerCallback, TriggerParam{PolicyID: id}); err != nil { SchedulerCallback, TriggerParam{PolicyID: id}, extras); err != nil {
return 0, err return 0, err
} }
@ -384,8 +385,9 @@ func (c *controller) UpdatePolicy(ctx context.Context, schema *policyModels.Sche
// schedule new // schedule new
if needSch { if needSch {
extras := make(map[string]interface{})
if _, err := c.scheduler.Schedule(ctx, job.P2PPreheat, schema.ID, "", cron, SchedulerCallback, if _, err := c.scheduler.Schedule(ctx, job.P2PPreheat, schema.ID, "", cron, SchedulerCallback,
TriggerParam{PolicyID: schema.ID}); err != nil { TriggerParam{PolicyID: schema.ID}, extras); err != nil {
return err return err
} }
} }

View File

@ -241,7 +241,7 @@ func (s *preheatSuite) TestCreatePolicy() {
FiltersStr: `[{"type":"repository","value":"harbor*"},{"type":"tag","value":"2*"}]`, FiltersStr: `[{"type":"repository","value":"harbor*"},{"type":"tag","value":"2*"}]`,
TriggerStr: fmt.Sprintf(`{"type":"%s", "trigger_setting":{"cron":"* * * * */1"}}`, policy.TriggerTypeScheduled), TriggerStr: fmt.Sprintf(`{"type":"%s", "trigger_setting":{"cron":"* * * * */1"}}`, policy.TriggerTypeScheduled),
} }
s.fakeScheduler.On("Schedule", s.ctx, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) s.fakeScheduler.On("Schedule", s.ctx, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
s.fakePolicyMgr.On("Create", s.ctx, policy).Return(int64(1), nil) s.fakePolicyMgr.On("Create", s.ctx, policy).Return(int64(1), nil)
s.fakePolicyMgr.On("Update", s.ctx, mock.Anything, mock.Anything).Return(nil) s.fakePolicyMgr.On("Update", s.ctx, mock.Anything, mock.Anything).Return(nil)
s.fakeScheduler.On("UnScheduleByVendor", s.ctx, mock.Anything, mock.Anything).Return(nil) s.fakeScheduler.On("UnScheduleByVendor", s.ctx, mock.Anything, mock.Anything).Return(nil)

View File

@ -122,9 +122,6 @@ func init() {
beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping") beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping")
beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List") beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List")
beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete") beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete")
beego.Router("/api/system/gc/:id", &GCAPI{}, "get:GetGC")
beego.Router("/api/system/gc/:id([0-9]+)/log", &GCAPI{}, "get:GetLog")
beego.Router("/api/system/gc/schedule", &GCAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/system/scanAll/schedule", &ScanAllAPI{}, "get:Get;put:Put;post:Post") beego.Router("/api/system/scanAll/schedule", &ScanAllAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/system/CVEAllowlist", &SysCVEAllowlistAPI{}, "get:Get;put:Put") beego.Router("/api/system/CVEAllowlist", &SysCVEAllowlistAPI{}, "get:Get;put:Put")
beego.Router("/api/system/oidc/ping", &OIDCAPI{}, "post:Ping") beego.Router("/api/system/oidc/ping", &OIDCAPI{}, "post:Ping")
@ -864,36 +861,6 @@ func (a testapi) DeleteMeta(authInfor usrInfo, projectID int64, name string) (in
return code, string(body), err return code, string(body), err
} }
func (a testapi) AddGC(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) {
_sling := sling.New().Post(a.basePath)
path := "/api/system/gc/schedule"
_sling = _sling.Path(path)
// body params
_sling = _sling.BodyJSON(adminReq)
var httpStatusCode int
var err error
httpStatusCode, _, err = request(_sling, jsonAcceptHeader, authInfor)
return httpStatusCode, err
}
func (a testapi) GCScheduleGet(authInfo usrInfo) (int, api_models.AdminJobSchedule, error) {
_sling := sling.New().Get(a.basePath)
path := "/api/system/gc/schedule"
_sling = _sling.Path(path)
httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo)
var successPayLoad api_models.AdminJobSchedule
if 200 == httpStatusCode && nil == err {
err = json.Unmarshal(body, &successPayLoad)
}
return httpStatusCode, successPayLoad, err
}
func (a testapi) AddScanAll(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) { func (a testapi) AddScanAll(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) {
_sling := sling.New().Post(a.basePath) _sling := sling.New().Post(a.basePath)

View File

@ -1,147 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"errors"
"github.com/goharbor/harbor/src/core/config"
"net/http"
"os"
"strconv"
common_job "github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/core/api/models"
)
// GCAPI handles request of harbor GC...
type GCAPI struct {
AJAPI
}
// Prepare validates the URL and parms, it needs the system admin permission.
func (gc *GCAPI) Prepare() {
gc.BaseController.Prepare()
if !gc.SecurityCtx.IsAuthenticated() {
gc.SendUnAuthorizedError(errors.New("UnAuthorized"))
return
}
if !gc.SecurityCtx.IsSysAdmin() {
gc.SendForbiddenError(errors.New(gc.SecurityCtx.GetUsername()))
return
}
}
// Post according to the request, it creates a cron schedule or a manual trigger for GC.
// create a daily schedule for GC
// {
// "schedule": {
// "type": "Daily",
// "cron": "0 0 0 * * *"
// },
// "parameters": {
// "delete_untagged": true
// }
// }
// create a manual trigger for GC
// {
// "schedule": {
// "type": "Manual"
// },
// "parameters": {
// "delete_untagged": true
// "read_only": true
// }
// }
func (gc *GCAPI) Post() {
parameters := make(map[string]interface{})
ajr := models.AdminJobReq{
Parameters: parameters,
}
isValid, err := gc.DecodeJSONReqAndValidate(&ajr)
if !isValid {
gc.SendBadRequestError(err)
return
}
ajr.Parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
// default is the non-blocking GC job.
ajr.Name = common_job.ImageGC
ajr.Parameters["time_window"] = config.GetGCTimeWindow()
// if specify read_only:true, API will submit the readonly GC job, otherwise default is non-blocking GC job.
readOnlyParam, exist := ajr.Parameters["read_only"]
if exist {
if readOnly, ok := readOnlyParam.(bool); ok && readOnly {
ajr.Name = common_job.ImageGCReadOnly
}
}
gc.submit(&ajr)
gc.Redirect(http.StatusCreated, strconv.FormatInt(ajr.ID, 10))
}
// Put handles GC cron schedule update/delete.
// Request: delete the schedule of GC
// {
// "schedule": {
// "type": "None",
// "cron": ""
// },
// "parameters": {
// "delete_untagged": true
// }
// }
func (gc *GCAPI) Put() {
parameters := make(map[string]interface{})
ajr := models.AdminJobReq{
Parameters: parameters,
}
isValid, err := gc.DecodeJSONReqAndValidate(&ajr)
if !isValid {
gc.SendBadRequestError(err)
return
}
ajr.Name = common_job.ImageGC
ajr.Parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
ajr.Parameters["time_window"] = config.GetGCTimeWindow()
gc.updateSchedule(ajr)
}
// GetGC ...
func (gc *GCAPI) GetGC() {
id, err := gc.GetInt64FromPath(":id")
if err != nil {
gc.SendInternalServerError(errors.New("need to specify gc id"))
return
}
gc.get(id)
}
// List returns the top 10 executions of GC which includes manual and cron.
func (gc *GCAPI) List() {
gc.list(common_job.ImageGC)
}
// Get gets GC schedule ...
func (gc *GCAPI) Get() {
gc.getSchedule(common_job.ImageGC)
}
// GetLog ...
func (gc *GCAPI) GetLog() {
id, err := gc.GetInt64FromPath(":id")
if err != nil {
gc.SendBadRequestError(errors.New("invalid ID"))
return
}
gc.getLog(id)
}

View File

@ -1,39 +0,0 @@
package api
import (
"testing"
"github.com/goharbor/harbor/src/testing/apitests/apilib"
"github.com/stretchr/testify/assert"
)
func TestGCPost(t *testing.T) {
adminJob001 := apilib.AdminJobReq{
Parameters: map[string]interface{}{"delete_untagged": false},
}
assert := assert.New(t)
apiTest := newHarborAPI()
// case 1: add a new admin job
code, err := apiTest.AddGC(*admin, adminJob001)
if err != nil {
t.Error("Error occurred while add a admin job", err.Error())
t.Log(err)
} else {
assert.Equal(201, code, "Add adminjob status should be 201")
}
}
func TestGCGet(t *testing.T) {
assert := assert.New(t)
apiTest := newHarborAPI()
code, _, err := apiTest.GCScheduleGet(*admin)
if err != nil {
t.Error("Error occurred while get a admin job", err.Error())
t.Log(err)
} else {
assert.Equal(200, code, "Get adminjob status should be 200")
}
}

View File

@ -93,10 +93,11 @@ func (r *DefaultAPIController) CreateRetention(p *policy.Metadata) (int64, error
if p.Trigger.Kind == policy.TriggerKindSchedule { if p.Trigger.Kind == policy.TriggerKindSchedule {
cron, ok := p.Trigger.Settings[policy.TriggerSettingsCron] cron, ok := p.Trigger.Settings[policy.TriggerSettingsCron]
if ok && len(cron.(string)) > 0 { if ok && len(cron.(string)) > 0 {
extras := make(map[string]interface{})
if _, err = r.scheduler.Schedule(orm.Context(), schedulerVendorType, id, "", cron.(string), SchedulerCallback, TriggerParam{ if _, err = r.scheduler.Schedule(orm.Context(), schedulerVendorType, id, "", cron.(string), SchedulerCallback, TriggerParam{
PolicyID: id, PolicyID: id,
Trigger: ExecutionTriggerSchedule, Trigger: ExecutionTriggerSchedule,
}); err != nil { }, extras); err != nil {
return 0, err return 0, err
} }
} }
@ -152,10 +153,11 @@ func (r *DefaultAPIController) UpdateRetention(p *policy.Metadata) error {
} }
} }
if needSch { if needSch {
extras := make(map[string]interface{})
_, err := r.scheduler.Schedule(orm.Context(), schedulerVendorType, p.ID, "", p.Trigger.Settings[policy.TriggerSettingsCron].(string), SchedulerCallback, TriggerParam{ _, err := r.scheduler.Schedule(orm.Context(), schedulerVendorType, p.ID, "", p.Trigger.Settings[policy.TriggerSettingsCron].(string), SchedulerCallback, TriggerParam{
PolicyID: p.ID, PolicyID: p.ID,
Trigger: ExecutionTriggerSchedule, Trigger: ExecutionTriggerSchedule,
}) }, extras)
if err != nil { if err != nil {
return err return err
} }

View File

@ -220,7 +220,7 @@ func (s *ControllerTestSuite) TestExecution() {
type fakeRetentionScheduler struct { type fakeRetentionScheduler struct {
} }
func (f *fakeRetentionScheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}) (int64, error) { func (f *fakeRetentionScheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
return 111, nil return 111, nil
} }

View File

@ -34,6 +34,7 @@ type schedule struct {
VendorID int64 `orm:"column(vendor_id)"` VendorID int64 `orm:"column(vendor_id)"`
CRONType string `orm:"column(cron_type)"` CRONType string `orm:"column(cron_type)"`
CRON string `orm:"column(cron)"` CRON string `orm:"column(cron)"`
ExtraAttrs string `orm:"column(extra_attrs)"`
CallbackFuncName string `orm:"column(callback_func_name)"` CallbackFuncName string `orm:"column(callback_func_name)"`
CallbackFuncParam string `orm:"column(callback_func_param)"` CallbackFuncParam string `orm:"column(callback_func_param)"`
CreationTime time.Time `orm:"column(creation_time)"` CreationTime time.Time `orm:"column(creation_time)"`

View File

@ -45,6 +45,7 @@ func (d *daoTestSuite) SetupTest() {
CRON: "0 * * * * *", CRON: "0 * * * * *",
CallbackFuncName: "callback_func_01", CallbackFuncName: "callback_func_01",
CallbackFuncParam: "callback_func_params", CallbackFuncParam: "callback_func_params",
ExtraAttrs: `{"key":"value"}`,
} }
id, err := d.dao.Create(d.ctx, schedule) id, err := d.dao.Create(d.ctx, schedule)
d.Require().Nil(err) d.Require().Nil(err)
@ -79,6 +80,7 @@ func (d *daoTestSuite) TestGet() {
schedule, err = d.dao.Get(d.ctx, d.id) schedule, err = d.dao.Get(d.ctx, d.id)
d.Require().Nil(err) d.Require().Nil(err)
d.Equal(d.id, schedule.ID) d.Equal(d.id, schedule.ID)
d.Equal("{\"key\":\"value\"}", schedule.ExtraAttrs)
} }
func (d *daoTestSuite) TestDelete() { func (d *daoTestSuite) TestDelete() {
@ -93,7 +95,7 @@ func (d *daoTestSuite) TestUpdate() {
// not found // not found
err := d.dao.Update(d.ctx, &schedule{ err := d.dao.Update(d.ctx, &schedule{
ID: 10000, ID: 10000,
}) }, "CRON")
d.True(errors.IsNotFoundErr(err)) d.True(errors.IsNotFoundErr(err))
// pass // pass

View File

@ -37,14 +37,15 @@ var (
// Schedule describes the detail information about the created schedule // Schedule describes the detail information about the created schedule
type Schedule struct { type Schedule struct {
ID int64 `json:"id"` ID int64 `json:"id"`
VendorType string `json:"vendor_type"` VendorType string `json:"vendor_type"`
VendorID int64 `json:"vendor_id"` VendorID int64 `json:"vendor_id"`
CRONType string `json:"cron_type"` CRONType string `json:"cron_type"`
CRON string `json:"cron"` CRON string `json:"cron"`
Status string `json:"status"` // status of the underlying task(jobservice job) ExtraAttrs map[string]interface{} `json:"extra_attrs"`
CreationTime time.Time `json:"creation_time"` Status string `json:"status"` // status of the underlying task(jobservice job)
UpdateTime time.Time `json:"update_time"` CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
// we can extend this model to include more information(e.g. how many times the schedule already // we can extend this model to include more information(e.g. how many times the schedule already
// runs; when will the schedule runs next time) // runs; when will the schedule runs next time)
} }
@ -59,7 +60,7 @@ type Scheduler interface {
// The "params" is passed to the callback function as encoded json string, so the callback // The "params" is passed to the callback function as encoded json string, so the callback
// function must decode it before using // function must decode it before using
Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
cron string, callbackFuncName string, params interface{}) (int64, error) cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error)
// UnScheduleByID the schedule specified by ID // UnScheduleByID the schedule specified by ID
UnScheduleByID(ctx context.Context, id int64) error UnScheduleByID(ctx context.Context, id int64) error
// UnScheduleByVendor the schedule specified by vendor // UnScheduleByVendor the schedule specified by vendor
@ -94,10 +95,10 @@ type scheduler struct {
// to out of control from the global transaction, and uses a new transaction that only // to out of control from the global transaction, and uses a new transaction that only
// covers the logic inside the function // covers the logic inside the function
func (s *scheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, func (s *scheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
cron string, callbackFuncName string, params interface{}) (int64, error) { cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
var scheduleID int64 var scheduleID int64
f := func(ctx context.Context) error { f := func(ctx context.Context) error {
id, err := s.schedule(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params) id, err := s.schedule(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
if err != nil { if err != nil {
return err return err
} }
@ -113,7 +114,7 @@ func (s *scheduler) Schedule(ctx context.Context, vendorType string, vendorID in
} }
func (s *scheduler) schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, func (s *scheduler) schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
cron string, callbackFuncName string, params interface{}) (int64, error) { cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
if len(vendorType) == 0 { if len(vendorType) == 0 {
return 0, fmt.Errorf("empty vendor type") return 0, fmt.Errorf("empty vendor type")
} }
@ -142,6 +143,13 @@ func (s *scheduler) schedule(ctx context.Context, vendorType string, vendorID in
} }
sched.CallbackFuncParam = string(paramsData) sched.CallbackFuncParam = string(paramsData)
} }
if extras != nil {
extrasData, err := json.Marshal(extras)
if err != nil {
return 0, err
}
sched.ExtraAttrs = string(extrasData)
}
// create schedule record // create schedule record
// when checkin hook comes, the database record must exist, // when checkin hook comes, the database record must exist,
// so the database record must be created first before submitting job // so the database record must be created first before submitting job
@ -272,6 +280,15 @@ func (s *scheduler) convertSchedule(ctx context.Context, schedule *schedule) (*S
CreationTime: schedule.CreationTime, CreationTime: schedule.CreationTime,
UpdateTime: schedule.UpdateTime, UpdateTime: schedule.UpdateTime,
} }
if len(schedule.ExtraAttrs) > 0 {
extras := map[string]interface{}{}
if err := json.Unmarshal([]byte(schedule.ExtraAttrs), &extras); err != nil {
log.Errorf("failed to unmarshal the extra attributes of schedule %d: %v", schedule.ID, err)
return nil, err
}
schd.ExtraAttrs = extras
}
executions, err := s.execMgr.List(ctx, &q.Query{ executions, err := s.execMgr.List(ctx, &q.Query{
Keywords: map[string]interface{}{ Keywords: map[string]interface{}{
"VendorType": JobNameScheduler, "VendorType": JobNameScheduler,

View File

@ -52,15 +52,16 @@ func (s *schedulerTestSuite) SetupTest() {
func (s *schedulerTestSuite) TestSchedule() { func (s *schedulerTestSuite) TestSchedule() {
// empty vendor type // empty vendor type
id, err := s.scheduler.Schedule(nil, "", 0, "", "0 * * * * *", "callback", nil) extras := make(map[string]interface{})
id, err := s.scheduler.Schedule(nil, "", 0, "", "0 * * * * *", "callback", nil, extras)
s.NotNil(err) s.NotNil(err)
// invalid cron // invalid cron
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "", "callback", nil) id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "", "callback", nil, extras)
s.NotNil(err) s.NotNil(err)
// callback function not exist // callback function not exist
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "not-exist", nil) id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "not-exist", nil, extras)
s.NotNil(err) s.NotNil(err)
// failed to submit to jobservice // failed to submit to jobservice
@ -73,7 +74,7 @@ func (s *schedulerTestSuite) TestSchedule() {
Status: job.ErrorStatus.String(), Status: job.ErrorStatus.String(),
}, nil) }, nil)
s.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil) s.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
_, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param") _, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param", extras)
s.Require().NotNil(err) s.Require().NotNil(err)
s.dao.AssertExpectations(s.T()) s.dao.AssertExpectations(s.T())
s.execMgr.AssertExpectations(s.T()) s.execMgr.AssertExpectations(s.T())
@ -91,7 +92,7 @@ func (s *schedulerTestSuite) TestSchedule() {
ExecutionID: 1, ExecutionID: 1,
Status: job.SuccessStatus.String(), Status: job.SuccessStatus.String(),
}, nil) }, nil)
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param") id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param", extras)
s.Require().Nil(err) s.Require().Nil(err)
s.Equal(int64(1), id) s.Equal(int64(1), id)
s.dao.AssertExpectations(s.T()) s.dao.AssertExpectations(s.T())

View File

@ -115,6 +115,22 @@ func (t *Task) GetStringFromExtraAttrs(key string) string {
return str return str
} }
// GetBoolFromExtraAttrs returns the bool value specified by key
func (t *Task) GetBoolFromExtraAttrs(key string) bool {
if len(t.ExtraAttrs) == 0 {
return false
}
rt, exist := t.ExtraAttrs[key]
if !exist {
return false
}
b, ok := rt.(bool)
if !ok {
return false
}
return b
}
// Job is the model represents the requested jobservice job // Job is the model represents the requested jobservice job
type Job struct { type Job struct {
Name string Name string

View File

@ -18,7 +18,7 @@
<clr-dg-cell>{{job.createTime | date:'medium'}}</clr-dg-cell> <clr-dg-cell>{{job.createTime | date:'medium'}}</clr-dg-cell>
<clr-dg-cell>{{job.updateTime | date:'medium'}}</clr-dg-cell> <clr-dg-cell>{{job.updateTime | date:'medium'}}</clr-dg-cell>
<clr-dg-cell> <clr-dg-cell>
<a *ngIf="job.status.toLowerCase() === 'finished' || job.status.toLowerCase() === 'error'" target="_blank" [href]="getLogLink(job.id)"><clr-icon shape="list"></clr-icon></a> <a *ngIf="job.status.toLowerCase() === 'success' || job.status.toLowerCase() === 'error'" target="_blank" [href]="getLogLink(job.id)"><clr-icon shape="list"></clr-icon></a>
</clr-dg-cell> </clr-dg-cell>
</clr-dg-row> </clr-dg-row>
<clr-dg-footer> <clr-dg-footer>

View File

@ -52,7 +52,8 @@ func (c *controller) Create(policy *model.Policy) (int64, error) {
return 0, err return 0, err
} }
if isScheduledTrigger(policy) { if isScheduledTrigger(policy) {
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, id, "", policy.Trigger.Settings.Cron, CallbackFuncName, id); err != nil { extras := make(map[string]interface{})
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, id, "", policy.Trigger.Settings.Cron, CallbackFuncName, id, extras); err != nil {
log.Errorf("failed to schedule the policy %d: %v", id, err) log.Errorf("failed to schedule the policy %d: %v", id, err)
} }
} }
@ -84,7 +85,8 @@ func (c *controller) Update(policy *model.Policy) error {
} }
// schedule again if needed // schedule again if needed
if isScheduledTrigger(policy) { if isScheduledTrigger(policy) {
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, policy.ID, "", policy.Trigger.Settings.Cron, CallbackFuncName, policy.ID); err != nil { extras := make(map[string]interface{})
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, policy.ID, "", policy.Trigger.Settings.Cron, CallbackFuncName, policy.ID, extras); err != nil {
return fmt.Errorf("failed to schedule the policy %d: %v", policy.ID, err) return fmt.Errorf("failed to schedule the policy %d: %v", policy.ID, err)
} }
} }

View File

@ -226,7 +226,7 @@ func TestCreate(t *testing.T) {
// scheduled trigger // scheduled trigger
scheduler.On("Schedule", mock.Anything, mock.Anything, scheduler.On("Schedule", mock.Anything, mock.Anything,
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
_, err = ctl.Create(&model.Policy{ _, err = ctl.Create(&model.Policy{
Enabled: true, Enabled: true,
Trigger: &model.Trigger{ Trigger: &model.Trigger{
@ -269,7 +269,7 @@ func TestUpdate(t *testing.T) {
// the trigger changed // the trigger changed
scheduler.On("Schedule", mock.Anything, mock.Anything, scheduler.On("Schedule", mock.Anything, mock.Anything,
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything, scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything,
mock.Anything).Return(nil) mock.Anything).Return(nil)

View File

@ -0,0 +1,198 @@
package handler
import (
"context"
"encoding/json"
"fmt"
"github.com/go-openapi/runtime/middleware"
"github.com/goharbor/harbor/src/controller/gc"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
"github.com/goharbor/harbor/src/server/v2.0/handler/model"
"github.com/goharbor/harbor/src/server/v2.0/models"
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/gc"
"os"
"strings"
)
type gcAPI struct {
BaseAPI
gcCtr gc.Controller
}
func newGCAPI() *gcAPI {
return &gcAPI{
gcCtr: gc.NewController(),
}
}
func (g *gcAPI) CreateGCSchedule(ctx context.Context, params operation.CreateGCScheduleParams) middleware.Responder {
id, err := g.kick(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters)
if err != nil {
return g.SendError(ctx, err)
}
// replace the /api/v2.0/system/gc/schedule/{id} to /api/v2.0/system/gc/{id}
lastSlashIndex := strings.LastIndex(params.HTTPRequest.URL.Path, "/")
if lastSlashIndex != -1 {
location := fmt.Sprintf("%s/%d", params.HTTPRequest.URL.Path[:lastSlashIndex], id)
return operation.NewCreateGCScheduleCreated().WithLocation(location)
}
return operation.NewCreateGCScheduleCreated()
}
func (g *gcAPI) UpdateGCSchedule(ctx context.Context, params operation.UpdateGCScheduleParams) middleware.Responder {
_, err := g.kick(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters)
if err != nil {
return g.SendError(ctx, err)
}
return operation.NewUpdateGCScheduleOK()
}
func (g *gcAPI) kick(ctx context.Context, scheType string, cron string, parameters map[string]interface{}) (int64, error) {
// set the required parameters for GC
parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
parameters["time_window"] = config.GetGCTimeWindow()
var err error
var id int64
switch scheType {
case ScheduleManual:
policy := gc.Policy{
ExtraAttrs: parameters,
}
if dryRun, ok := parameters["dry_run"].(bool); ok {
policy.DryRun = dryRun
}
if deleteUntagged, ok := parameters["delete_untagged"].(bool); ok {
policy.DeleteUntagged = deleteUntagged
}
id, err = g.gcCtr.Start(ctx, policy, task.ExecutionTriggerManual)
case ScheduleNone:
err = g.gcCtr.DeleteSchedule(ctx)
case ScheduleHourly, ScheduleDaily, ScheduleWeekly, ScheduleCustom:
policy := gc.Policy{
ExtraAttrs: parameters,
}
if dryRun, ok := parameters["dry_run"].(bool); ok {
policy.DryRun = dryRun
}
if deleteUntagged, ok := parameters["delete_untagged"].(bool); ok {
policy.DeleteUntagged = deleteUntagged
}
err = g.updateSchedule(ctx, scheType, cron, policy)
}
return id, err
}
func (g *gcAPI) createSchedule(ctx context.Context, cronType, cron string, policy gc.Policy) error {
if cron == "" {
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("empty cron string for gc schedule")
}
_, err := g.gcCtr.CreateSchedule(ctx, cronType, cron, policy)
if err != nil {
return err
}
return nil
}
func (g *gcAPI) updateSchedule(ctx context.Context, cronType, cron string, policy gc.Policy) error {
if err := g.gcCtr.DeleteSchedule(ctx); err != nil {
return err
}
return g.createSchedule(ctx, cronType, cron, policy)
}
func (g *gcAPI) GetGCSchedule(ctx context.Context, params operation.GetGCScheduleParams) middleware.Responder {
schedule, err := g.gcCtr.GetSchedule(ctx)
if errors.IsNotFoundErr(err) {
return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(&scheduler.Schedule{}).ToSwagger())
}
if err != nil {
return g.SendError(ctx, err)
}
return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(schedule).ToSwagger())
}
func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryParams) middleware.Responder {
query, err := g.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
if err != nil {
return g.SendError(ctx, err)
}
total, err := g.gcCtr.ExecutionCount(ctx, query)
if err != nil {
return g.SendError(ctx, err)
}
execs, err := g.gcCtr.ListExecutions(ctx, query)
if err != nil {
return g.SendError(ctx, err)
}
var hs []*model.GCHistory
for _, exec := range execs {
extraAttrsString, err := json.Marshal(exec.ExtraAttrs)
if err != nil {
return g.SendError(ctx, err)
}
hs = append(hs, &model.GCHistory{
ID: exec.ID,
Name: gc.GCVendorType,
Kind: exec.Trigger,
Parameters: string(extraAttrsString),
Schedule: &model.ScheduleParam{
Type: exec.Trigger,
},
Status: exec.Status,
CreationTime: exec.StartTime,
UpdateTime: exec.EndTime,
})
}
var results []*models.GCHistory
for _, h := range hs {
results = append(results, h.ToSwagger())
}
return operation.NewGetGCHistoryOK().
WithXTotalCount(total).
WithLink(g.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
WithPayload(results)
}
func (g *gcAPI) GetGC(ctx context.Context, params operation.GetGCParams) middleware.Responder {
exec, err := g.gcCtr.GetExecution(ctx, params.GcID)
if err != nil {
return g.SendError(ctx, err)
}
extraAttrsString, err := json.Marshal(exec.ExtraAttrs)
if err != nil {
return g.SendError(ctx, err)
}
res := &model.GCHistory{
ID: exec.ID,
Name: gc.GCVendorType,
Kind: exec.Trigger,
Parameters: string(extraAttrsString),
Status: exec.Status,
Schedule: &model.ScheduleParam{
Type: exec.Trigger,
},
CreationTime: exec.StartTime,
UpdateTime: exec.EndTime,
}
return operation.NewGetGCOK().WithPayload(res.ToSwagger())
}
func (g *gcAPI) GetGCLog(ctx context.Context, params operation.GetGCLogParams) middleware.Responder {
log, err := g.gcCtr.GetTaskLog(ctx, params.GcID)
if err != nil {
return g.SendError(ctx, err)
}
return operation.NewGetGCLogOK().WithPayload(string(log))
}

View File

@ -41,6 +41,7 @@ func New() http.Handler {
ReplicationAPI: newReplicationAPI(), ReplicationAPI: newReplicationAPI(),
SysteminfoAPI: newSystemInfoAPI(), SysteminfoAPI: newSystemInfoAPI(),
PingAPI: newPingAPI(), PingAPI: newPingAPI(),
GcAPI: newGCAPI(),
}) })
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)

View File

@ -0,0 +1,84 @@
package model
import (
"encoding/json"
"github.com/go-openapi/strfmt"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/server/v2.0/models"
"time"
)
// ScheduleParam defines the parameter of schedule trigger
type ScheduleParam struct {
// Daily, Weekly, Custom, Manual, None
Type string `json:"type"`
// The cron string of scheduled job
Cron string `json:"cron"`
}
// GCHistory gc execution history
type GCHistory struct {
Schedule *ScheduleParam `json:"schedule"`
ID int64 `json:"id"`
Name string `json:"job_name"`
Kind string `json:"job_kind"`
Parameters string `json:"job_parameters"`
Status string `json:"job_status"`
UUID string `json:"-"`
Deleted bool `json:"deleted"`
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
}
// ToSwagger converts the history to the swagger model
func (h *GCHistory) ToSwagger() *models.GCHistory {
return &models.GCHistory{
ID: h.ID,
JobName: h.Name,
JobKind: h.Kind,
JobParameters: h.Parameters,
Deleted: h.Deleted,
JobStatus: h.Status,
Schedule: &models.ScheduleObj{
Cron: h.Schedule.Cron,
Type: h.Schedule.Type,
},
CreationTime: strfmt.DateTime(h.CreationTime),
UpdateTime: strfmt.DateTime(h.UpdateTime),
}
}
// Schedule ...
type Schedule struct {
*scheduler.Schedule
}
// ToSwagger converts the schedule to the swagger model
// TODO remove the hard code when after issue https://github.com/goharbor/harbor/issues/13047 is resolved.
func (s *Schedule) ToSwagger() *models.GCHistory {
e, err := json.Marshal(s.ExtraAttrs)
if err != nil {
log.Error(err)
}
return &models.GCHistory{
ID: 0,
JobName: "",
JobKind: s.CRON,
JobParameters: string(e),
Deleted: false,
JobStatus: "",
Schedule: &models.ScheduleObj{
Cron: s.CRON,
Type: "Custom",
},
CreationTime: strfmt.DateTime(s.CreationTime),
UpdateTime: strfmt.DateTime(s.UpdateTime),
}
}
// NewSchedule ...
func NewSchedule(s *scheduler.Schedule) *Schedule {
return &Schedule{Schedule: s}
}

View File

@ -29,6 +29,21 @@ import (
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
) )
const (
// ScheduleHourly : 'Hourly'
ScheduleHourly = "Hourly"
// ScheduleDaily : 'Daily'
ScheduleDaily = "Daily"
// ScheduleWeekly : 'Weekly'
ScheduleWeekly = "Weekly"
// ScheduleCustom : 'Custom'
ScheduleCustom = "Custom"
// ScheduleManual : 'Manual'
ScheduleManual = "Manual"
// ScheduleNone : 'None'
ScheduleNone = "None"
)
func boolValue(v *bool) bool { func boolValue(v *bool) bool {
if v != nil { if v != nil {
return *v return *v

View File

@ -45,10 +45,6 @@ func registerLegacyRoutes() {
beego.Router("/api/"+version+"/quotas", &api.QuotaAPI{}, "get:List") beego.Router("/api/"+version+"/quotas", &api.QuotaAPI{}, "get:List")
beego.Router("/api/"+version+"/quotas/:id([0-9]+)", &api.QuotaAPI{}, "get:Get;put:Put") beego.Router("/api/"+version+"/quotas/:id([0-9]+)", &api.QuotaAPI{}, "get:Get;put:Put")
beego.Router("/api/"+version+"/system/gc", &api.GCAPI{}, "get:List")
beego.Router("/api/"+version+"/system/gc/:id", &api.GCAPI{}, "get:GetGC")
beego.Router("/api/"+version+"/system/gc/:id([0-9]+)/log", &api.GCAPI{}, "get:GetLog")
beego.Router("/api/"+version+"/system/gc/schedule", &api.GCAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/"+version+"/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post") beego.Router("/api/"+version+"/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/"+version+"/system/CVEAllowlist", &api.SysCVEAllowlistAPI{}, "get:Get;put:Put") beego.Router("/api/"+version+"/system/CVEAllowlist", &api.SysCVEAllowlistAPI{}, "get:Get;put:Put")
beego.Router("/api/"+version+"/system/oidc/ping", &api.OIDCAPI{}, "post:Ping") beego.Router("/api/"+version+"/system/oidc/ping", &api.OIDCAPI{}, "post:Ping")

View File

@ -62,20 +62,20 @@ func (_m *Scheduler) ListSchedules(ctx context.Context, query *q.Query) ([]*sche
return r0, r1 return r0, r1
} }
// Schedule provides a mock function with given fields: ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params // Schedule provides a mock function with given fields: ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras
func (_m *Scheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}) (int64, error) { func (_m *Scheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
ret := _m.Called(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params) ret := _m.Called(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
var r0 int64 var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, string, int64, string, string, string, interface{}) int64); ok { if rf, ok := ret.Get(0).(func(context.Context, string, int64, string, string, string, interface{}, map[string]interface{}) int64); ok {
r0 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params) r0 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
} else { } else {
r0 = ret.Get(0).(int64) r0 = ret.Get(0).(int64)
} }
var r1 error var r1 error
if rf, ok := ret.Get(1).(func(context.Context, string, int64, string, string, string, interface{}) error); ok { if rf, ok := ret.Get(1).(func(context.Context, string, int64, string, string, string, interface{}, map[string]interface{}) error); ok {
r1 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params) r1 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
} else { } else {
r1 = ret.Error(1) r1 = ret.Error(1)
} }

View File

@ -61,6 +61,7 @@ def _create_client(server, credential, debug, api_type="products"):
"scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)), "scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)),
"replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)), "replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)),
"robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)), "robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)),
"gc": v2_swagger_client.GcApi(v2_swagger_client.ApiClient(cfg)),
}.get(api_type,'Error: Wrong API type') }.get(api_type,'Error: Wrong API type')
def _assert_status_code(expect_code, return_code): def _assert_status_code(expect_code, return_code):

View File

@ -0,0 +1,134 @@
# -*- coding: utf-8 -*-
import time
import base
import re
import v2_swagger_client
from v2_swagger_client.rest import ApiException
class GC(base.Base, object):
def __init__(self):
super(GC,self).__init__(api_type = "gc")
def get_gc_history(self, expect_status_code = 200, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
try:
data, status_code, _ = client.get_gc_history_with_http_info()
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return data
def get_gc_status_by_id(self, job_id, expect_status_code = 200, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
try:
data, status_code, _ = client.get_gc_with_http_info(job_id)
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return data
def get_gc_log_by_id(self, job_id, expect_status_code = 200, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
try:
data, status_code, _ = client.get_gc_log_with_http_info(job_id)
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return data
def get_gc_schedule(self, expect_status_code = 200, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
try:
data, status_code, _ = client.get_gc_schedule_with_http_info()
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return data
def create_gc_schedule(self, schedule_type, is_delete_untagged, cron = None, expect_status_code = 201, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
gc_parameters = {'delete_untagged':is_delete_untagged}
gc_schedule = v2_swagger_client.ScheduleObj()
gc_schedule.type = schedule_type
if cron is not None:
gc_schedule.cron = cron
gc_job = v2_swagger_client.Schedule()
gc_job.schedule = gc_schedule
gc_job.parameters = gc_parameters
try:
_, status_code, header = client.create_gc_schedule_with_http_info(gc_job)
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Create GC schedule response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Create GC schedule result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return base._get_id_from_header(header)
def gc_now(self, is_delete_untagged=False, **kwargs):
gc_id = self.create_gc_schedule('Manual', is_delete_untagged, **kwargs)
return gc_id
def validate_gc_job_status(self, gc_id, expected_gc_status, **kwargs):
get_gc_status_finish = False
timeout_count = 20
while timeout_count > 0:
time.sleep(5)
status = self.get_gc_status_by_id(gc_id, **kwargs)
print("GC job No: {}, status: {}".format(timeout_count, status.job_status))
if status.job_status == expected_gc_status:
get_gc_status_finish = True
break
timeout_count = timeout_count - 1
if not (get_gc_status_finish):
raise Exception("GC status is not as expected '{}' actual GC status is '{}'".format(expected_gc_status, status.job_status))
def validate_deletion_success(self, gc_id, **kwargs):
log_content = self.get_gc_log_by_id(gc_id, **kwargs)
key_message = "manifests eligible for deletion"
key_message_pos = log_content.find(key_message)
full_message = log_content[key_message_pos-30 : key_message_pos + len(key_message)]
deleted_files_count_list = re.findall(r'\s+(\d+)\s+blobs\s+and\s+\d+\s+manifests\s+eligible\s+for\s+deletion', full_message)
if len(deleted_files_count_list) != 1:
raise Exception(r"Fail to get blobs eligible for deletion in log file, failure is {}.".format(len(deleted_files_count_list)))
deleted_files_count = int(deleted_files_count_list[0])
if deleted_files_count == 0:
raise Exception(r"Get blobs eligible for deletion count is {}, while we expect more than 1.".format(deleted_files_count))

View File

@ -157,38 +157,6 @@ class System(base.Base):
scan_all_id = self.create_scan_all_schedule('Manual', **kwargs) scan_all_id = self.create_scan_all_schedule('Manual', **kwargs)
return scan_all_id return scan_all_id
def gc_now(self, is_delete_untagged=False, **kwargs):
gc_id = self.create_gc_schedule('Manual', is_delete_untagged, **kwargs)
return gc_id
def validate_gc_job_status(self, gc_id, expected_gc_status, **kwargs):
get_gc_status_finish = False
timeout_count = 20
while timeout_count > 0:
time.sleep(5)
status = self.get_gc_status_by_id(gc_id, **kwargs)
print("GC job No: {}, status: {}".format(timeout_count, status.job_status))
if status.job_status == expected_gc_status:
get_gc_status_finish = True
break
timeout_count = timeout_count - 1
if not (get_gc_status_finish):
raise Exception("GC status is not as expected '{}' actual GC status is '{}'".format(expected_gc_status, status.job_status))
def validate_deletion_success(self, gc_id, **kwargs):
log_content = self.get_gc_log_by_id(gc_id, **kwargs)
key_message = "manifests eligible for deletion"
key_message_pos = log_content.find(key_message)
full_message = log_content[key_message_pos-30 : key_message_pos + len(key_message)]
deleted_files_count_list = re.findall(r'\s+(\d+)\s+blobs\s+and\s+\d+\s+manifests\s+eligible\s+for\s+deletion', full_message)
if len(deleted_files_count_list) != 1:
raise Exception(r"Fail to get blobs eligible for deletion in log file, failure is {}.".format(len(deleted_files_count_list)))
deleted_files_count = int(deleted_files_count_list[0])
if deleted_files_count == 0:
raise Exception(r"Get blobs eligible for deletion count is {}, while we expect more than 1.".format(deleted_files_count))
def set_cve_allowlist(self, expires_at=None, expected_status_code=200, *cve_ids, **kwargs): def set_cve_allowlist(self, expires_at=None, expected_status_code=200, *cve_ids, **kwargs):
client = self._get_client(**kwargs) client = self._get_client(**kwargs)
cve_list = [swagger_client.CVEAllowlistItem(cve_id=c) for c in cve_ids] cve_list = [swagger_client.CVEAllowlistItem(cve_id=c) for c in cve_ids]

View File

@ -7,17 +7,17 @@ from testutils import ADMIN_CLIENT, suppress_urllib3_warning
from testutils import TEARDOWN from testutils import TEARDOWN
from testutils import harbor_server from testutils import harbor_server
from library.user import User from library.user import User
from library.system import System
from library.project import Project from library.project import Project
from library.repository import Repository from library.repository import Repository
from library.base import _assert_status_code from library.base import _assert_status_code
from library.repository import push_special_image_to_project from library.repository import push_special_image_to_project
from library.artifact import Artifact from library.artifact import Artifact
from library.gc import GC
class TestProjects(unittest.TestCase): class TestProjects(unittest.TestCase):
@suppress_urllib3_warning @suppress_urllib3_warning
def setUp(self): def setUp(self):
self.system = System() self.gc = GC()
self.project = Project() self.project = Project()
self.user = User() self.user = User()
self.repo = Repository() self.repo = Repository()
@ -82,13 +82,13 @@ class TestProjects(unittest.TestCase):
self.artifact.delete_tag(TestProjects.project_gc_untag_name, self.repo_name_untag, self.tag, self.tag, **ADMIN_CLIENT) self.artifact.delete_tag(TestProjects.project_gc_untag_name, self.repo_name_untag, self.tag, self.tag, **ADMIN_CLIENT)
#5. Tigger garbage collection operation; #5. Tigger garbage collection operation;
gc_id = self.system.gc_now(**ADMIN_CLIENT) gc_id = self.gc.gc_now(**ADMIN_CLIENT)
#6. Check garbage collection job was finished; #6. Check garbage collection job was finished;
self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT) self.gc.validate_gc_job_status(gc_id, "Success", **ADMIN_CLIENT)
#7. Get garbage collection log, check there is a number of files was deleted; #7. Get garbage collection log, check there is a number of files was deleted;
self.system.validate_deletion_success(gc_id, **ADMIN_CLIENT) self.gc.validate_deletion_success(gc_id, **ADMIN_CLIENT)
artifacts = self.artifact.list_artifacts(TestProjects.project_gc_untag_name, self.repo_name_untag, **TestProjects.USER_GC_CLIENT) artifacts = self.artifact.list_artifacts(TestProjects.project_gc_untag_name, self.repo_name_untag, **TestProjects.USER_GC_CLIENT)
_assert_status_code(len(artifacts), 1) _assert_status_code(len(artifacts), 1)
@ -96,13 +96,13 @@ class TestProjects(unittest.TestCase):
time.sleep(5) time.sleep(5)
#9. Tigger garbage collection operation; #9. Tigger garbage collection operation;
gc_id = self.system.gc_now(is_delete_untagged=True, **ADMIN_CLIENT) gc_id = self.gc.gc_now(is_delete_untagged=True, **ADMIN_CLIENT)
#10. Check garbage collection job was finished; #10. Check garbage collection job was finished;
self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT) self.gc.validate_gc_job_status(gc_id, "Success", **ADMIN_CLIENT)
#7. Get garbage collection log, check there is a number of files was deleted; #7. Get garbage collection log, check there is a number of files was deleted;
self.system.validate_deletion_success(gc_id, **ADMIN_CLIENT) self.gc.validate_deletion_success(gc_id, **ADMIN_CLIENT)
#11. Repository with untag image should be still there; #11. Repository with untag image should be still there;
repo_data_untag = self.repo.list_repositories(TestProjects.project_gc_untag_name, **TestProjects.USER_GC_CLIENT) repo_data_untag = self.repo.list_repositories(TestProjects.project_gc_untag_name, **TestProjects.USER_GC_CLIENT)