mirror of
https://github.com/goharbor/harbor.git
synced 2025-01-11 18:38:14 +01:00
Merge pull request #13139 from wy65701436/migrate-gc
Migrate gc to task manager
This commit is contained in:
commit
6569016d35
@ -1549,25 +1549,6 @@ paths:
|
|||||||
description: Only admin has this authority.
|
description: Only admin has this authority.
|
||||||
'500':
|
'500':
|
||||||
description: Unexpected internal errors.
|
description: Unexpected internal errors.
|
||||||
/system/gc:
|
|
||||||
get:
|
|
||||||
summary: Get gc results.
|
|
||||||
description: This endpoint let user get latest ten gc results.
|
|
||||||
tags:
|
|
||||||
- Products
|
|
||||||
responses:
|
|
||||||
'200':
|
|
||||||
description: Get gc results successfully.
|
|
||||||
schema:
|
|
||||||
type: array
|
|
||||||
items:
|
|
||||||
$ref: '#/definitions/GCResult'
|
|
||||||
'401':
|
|
||||||
description: User need to log in first.
|
|
||||||
'403':
|
|
||||||
description: User does not have permission of admin role.
|
|
||||||
'500':
|
|
||||||
description: Unexpected internal errors.
|
|
||||||
'/system/gc/{id}':
|
'/system/gc/{id}':
|
||||||
get:
|
get:
|
||||||
summary: Get gc status.
|
summary: Get gc status.
|
||||||
@ -1592,101 +1573,6 @@ paths:
|
|||||||
description: User does not have permission of admin role.
|
description: User does not have permission of admin role.
|
||||||
'500':
|
'500':
|
||||||
description: Unexpected internal errors.
|
description: Unexpected internal errors.
|
||||||
'/system/gc/{id}/log':
|
|
||||||
get:
|
|
||||||
summary: Get gc job log.
|
|
||||||
description: This endpoint let user get gc job logs filtered by specific ID.
|
|
||||||
parameters:
|
|
||||||
- name: id
|
|
||||||
in: path
|
|
||||||
type: integer
|
|
||||||
format: int64
|
|
||||||
required: true
|
|
||||||
description: Relevant job ID
|
|
||||||
tags:
|
|
||||||
- Products
|
|
||||||
responses:
|
|
||||||
'200':
|
|
||||||
description: Get successfully.
|
|
||||||
schema:
|
|
||||||
type: string
|
|
||||||
'400':
|
|
||||||
description: Illegal format of provided ID value.
|
|
||||||
'401':
|
|
||||||
description: User need to log in first.
|
|
||||||
'403':
|
|
||||||
description: User does not have permission of admin role.
|
|
||||||
'404':
|
|
||||||
description: The specific gc ID's log does not exist.
|
|
||||||
'500':
|
|
||||||
description: Unexpected internal errors.
|
|
||||||
/system/gc/schedule:
|
|
||||||
get:
|
|
||||||
summary: Get gc's schedule.
|
|
||||||
description: This endpoint is for get schedule of gc job.
|
|
||||||
tags:
|
|
||||||
- Products
|
|
||||||
responses:
|
|
||||||
'200':
|
|
||||||
description: Get gc's schedule.
|
|
||||||
schema:
|
|
||||||
$ref: '#/definitions/AdminJobSchedule'
|
|
||||||
'401':
|
|
||||||
description: User need to log in first.
|
|
||||||
'403':
|
|
||||||
description: Only admin has this authority.
|
|
||||||
'500':
|
|
||||||
description: Unexpected internal errors.
|
|
||||||
put:
|
|
||||||
summary: Update gc's schedule.
|
|
||||||
description: |
|
|
||||||
This endpoint is for update gc schedule.
|
|
||||||
parameters:
|
|
||||||
- name: schedule
|
|
||||||
in: body
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
$ref: '#/definitions/AdminJobSchedule'
|
|
||||||
description: Updates of gc's schedule.
|
|
||||||
tags:
|
|
||||||
- Products
|
|
||||||
responses:
|
|
||||||
'200':
|
|
||||||
description: Updated gc's schedule successfully.
|
|
||||||
'400':
|
|
||||||
description: Invalid schedule type.
|
|
||||||
'401':
|
|
||||||
description: User need to log in first.
|
|
||||||
'403':
|
|
||||||
description: User does not have permission of admin role.
|
|
||||||
'500':
|
|
||||||
description: Unexpected internal errors.
|
|
||||||
post:
|
|
||||||
summary: Create a gc schedule.
|
|
||||||
description: |
|
|
||||||
This endpoint is for update gc schedule.
|
|
||||||
parameters:
|
|
||||||
- name: schedule
|
|
||||||
in: body
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
$ref: '#/definitions/AdminJobSchedule'
|
|
||||||
description: Updates of gc's schedule.
|
|
||||||
tags:
|
|
||||||
- Products
|
|
||||||
responses:
|
|
||||||
'200':
|
|
||||||
description: GC schedule successfully.
|
|
||||||
'400':
|
|
||||||
description: Invalid schedule type.
|
|
||||||
'401':
|
|
||||||
description: User need to log in first.
|
|
||||||
'403':
|
|
||||||
description: User does not have permission of admin role.
|
|
||||||
'409':
|
|
||||||
description: There is a "gc" job in progress, so the request cannot be served.
|
|
||||||
'500':
|
|
||||||
description: Unexpected internal errors.
|
|
||||||
/system/scanAll/schedule:
|
/system/scanAll/schedule:
|
||||||
get:
|
get:
|
||||||
summary: Get scan_all's schedule.
|
summary: Get scan_all's schedule.
|
||||||
|
@ -2017,6 +2017,155 @@ paths:
|
|||||||
description: Not found the default root certificate.
|
description: Not found the default root certificate.
|
||||||
'500':
|
'500':
|
||||||
$ref: '#/responses/500'
|
$ref: '#/responses/500'
|
||||||
|
/system/gc:
|
||||||
|
get:
|
||||||
|
summary: Get gc results.
|
||||||
|
description: This endpoint let user get gc execution history.
|
||||||
|
tags:
|
||||||
|
- gc
|
||||||
|
operationId: getGCHistory
|
||||||
|
parameters:
|
||||||
|
- $ref: '#/parameters/query'
|
||||||
|
- $ref: '#/parameters/page'
|
||||||
|
- $ref: '#/parameters/pageSize'
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: Get gc results successfully.
|
||||||
|
headers:
|
||||||
|
X-Total-Count:
|
||||||
|
description: The total count of history
|
||||||
|
type: integer
|
||||||
|
Link:
|
||||||
|
description: Link refers to the previous page and next page
|
||||||
|
type: string
|
||||||
|
schema:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
$ref: '#/definitions/GCHistory'
|
||||||
|
'401':
|
||||||
|
$ref: '#/responses/401'
|
||||||
|
'403':
|
||||||
|
$ref: '#/responses/403'
|
||||||
|
'500':
|
||||||
|
$ref: '#/responses/500'
|
||||||
|
/system/gc/{gc_id}:
|
||||||
|
get:
|
||||||
|
summary: Get gc status.
|
||||||
|
description: This endpoint let user get gc status filtered by specific ID.
|
||||||
|
operationId: getGC
|
||||||
|
parameters:
|
||||||
|
- $ref: '#/parameters/gcId'
|
||||||
|
tags:
|
||||||
|
- gc
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: Get gc results successfully.
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/GCHistory'
|
||||||
|
'401':
|
||||||
|
$ref: '#/responses/401'
|
||||||
|
'403':
|
||||||
|
$ref: '#/responses/403'
|
||||||
|
'404':
|
||||||
|
$ref: '#/responses/404'
|
||||||
|
'500':
|
||||||
|
$ref: '#/responses/500'
|
||||||
|
/system/gc/{gc_id}/log:
|
||||||
|
get:
|
||||||
|
summary: Get gc job log.
|
||||||
|
description: This endpoint let user get gc job logs filtered by specific ID.
|
||||||
|
operationId: getGCLog
|
||||||
|
parameters:
|
||||||
|
- $ref: '#/parameters/gcId'
|
||||||
|
tags:
|
||||||
|
- gc
|
||||||
|
produces:
|
||||||
|
- text/plain
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: Get successfully.
|
||||||
|
schema:
|
||||||
|
type: string
|
||||||
|
'400':
|
||||||
|
$ref: '#/responses/400'
|
||||||
|
'401':
|
||||||
|
$ref: '#/responses/401'
|
||||||
|
'403':
|
||||||
|
$ref: '#/responses/403'
|
||||||
|
'404':
|
||||||
|
$ref: '#/responses/404'
|
||||||
|
'500':
|
||||||
|
$ref: '#/responses/500'
|
||||||
|
/system/gc/schedule:
|
||||||
|
get:
|
||||||
|
summary: Get gc's schedule.
|
||||||
|
description: This endpoint is for get schedule of gc job.
|
||||||
|
operationId: getGCSchedule
|
||||||
|
tags:
|
||||||
|
- gc
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: Get gc's schedule.
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/GCHistory'
|
||||||
|
'401':
|
||||||
|
$ref: '#/responses/401'
|
||||||
|
'403':
|
||||||
|
$ref: '#/responses/403'
|
||||||
|
'500':
|
||||||
|
$ref: '#/responses/500'
|
||||||
|
post:
|
||||||
|
summary: Create a gc schedule.
|
||||||
|
description: |
|
||||||
|
This endpoint is for update gc schedule.
|
||||||
|
operationId: createGCSchedule
|
||||||
|
parameters:
|
||||||
|
- name: schedule
|
||||||
|
in: body
|
||||||
|
required: true
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/Schedule'
|
||||||
|
description: Updates of gc's schedule.
|
||||||
|
tags:
|
||||||
|
- gc
|
||||||
|
responses:
|
||||||
|
'201':
|
||||||
|
$ref: '#/responses/201'
|
||||||
|
'400':
|
||||||
|
$ref: '#/responses/400'
|
||||||
|
'401':
|
||||||
|
$ref: '#/responses/401'
|
||||||
|
'403':
|
||||||
|
$ref: '#/responses/403'
|
||||||
|
'409':
|
||||||
|
$ref: '#/responses/409'
|
||||||
|
'500':
|
||||||
|
$ref: '#/responses/500'
|
||||||
|
put:
|
||||||
|
summary: Update gc's schedule.
|
||||||
|
description: |
|
||||||
|
This endpoint is for update gc schedule.
|
||||||
|
operationId: updateGCSchedule
|
||||||
|
parameters:
|
||||||
|
- name: schedule
|
||||||
|
in: body
|
||||||
|
required: true
|
||||||
|
schema:
|
||||||
|
$ref: '#/definitions/Schedule'
|
||||||
|
description: Updates of gc's schedule.
|
||||||
|
tags:
|
||||||
|
- gc
|
||||||
|
responses:
|
||||||
|
'200':
|
||||||
|
description: Updated gc's schedule successfully.
|
||||||
|
'400':
|
||||||
|
$ref: '#/responses/400'
|
||||||
|
'401':
|
||||||
|
$ref: '#/responses/401'
|
||||||
|
'403':
|
||||||
|
$ref: '#/responses/403'
|
||||||
|
'500':
|
||||||
|
$ref: '#/responses/500'
|
||||||
/ping:
|
/ping:
|
||||||
get:
|
get:
|
||||||
summary: Ping Harbor to check if it's alive.
|
summary: Ping Harbor to check if it's alive.
|
||||||
@ -2134,6 +2283,13 @@ parameters:
|
|||||||
description: Robot ID
|
description: Robot ID
|
||||||
required: true
|
required: true
|
||||||
type: integer
|
type: integer
|
||||||
|
gcId:
|
||||||
|
name: gc_id
|
||||||
|
in: path
|
||||||
|
description: The ID of the gc log
|
||||||
|
required: true
|
||||||
|
type: integer
|
||||||
|
format: int64
|
||||||
responses:
|
responses:
|
||||||
'200':
|
'200':
|
||||||
description: Success
|
description: Success
|
||||||
@ -3351,3 +3507,55 @@ definitions:
|
|||||||
description: The storage of system.
|
description: The storage of system.
|
||||||
items:
|
items:
|
||||||
$ref: '#/definitions/Storage'
|
$ref: '#/definitions/Storage'
|
||||||
|
GCHistory:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
id:
|
||||||
|
type: integer
|
||||||
|
description: the id of gc job.
|
||||||
|
job_name:
|
||||||
|
type: string
|
||||||
|
description: the job name of gc job.
|
||||||
|
job_kind:
|
||||||
|
type: string
|
||||||
|
description: the job kind of gc job.
|
||||||
|
job_parameters:
|
||||||
|
type: string
|
||||||
|
description: the job parameters of gc job.
|
||||||
|
schedule:
|
||||||
|
$ref: '#/definitions/ScheduleObj'
|
||||||
|
job_status:
|
||||||
|
type: string
|
||||||
|
description: the status of gc job.
|
||||||
|
deleted:
|
||||||
|
type: boolean
|
||||||
|
description: if gc job was deleted.
|
||||||
|
creation_time:
|
||||||
|
type: string
|
||||||
|
format: date-time
|
||||||
|
description: the creation time of gc job.
|
||||||
|
update_time:
|
||||||
|
type: string
|
||||||
|
format: date-time
|
||||||
|
description: the update time of gc job.
|
||||||
|
Schedule:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
schedule:
|
||||||
|
$ref: '#/definitions/ScheduleObj'
|
||||||
|
parameters:
|
||||||
|
type: object
|
||||||
|
description: The parameters of admin job
|
||||||
|
additionalProperties:
|
||||||
|
type: object
|
||||||
|
ScheduleObj:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
type:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manually' and 'None'.
|
||||||
|
'Manually' means to trigger it right away and 'None' means to cancel the schedule.
|
||||||
|
cron:
|
||||||
|
type: string
|
||||||
|
description: A cron expression, a time-based job scheduler.
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
/*
|
/*
|
||||||
Fixes issue https://github.com/goharbor/harbor/issues/13317
|
Fixes issue https://github.com/goharbor/harbor/issues/13317
|
||||||
Ensure the role_id of maintainer is 4 and the role_id of limisted guest is 5
|
Ensure the role_id of maintainer is 4 and the role_id of limisted guest is 5
|
||||||
*/
|
*/
|
||||||
@ -268,3 +268,7 @@ BEGIN
|
|||||||
UPDATE scanner_registration SET is_default = TRUE WHERE name = 'Trivy' AND immutable = TRUE;
|
UPDATE scanner_registration SET is_default = TRUE WHERE name = 'Trivy' AND immutable = TRUE;
|
||||||
END IF;
|
END IF;
|
||||||
END $$;
|
END $$;
|
||||||
|
ALTER TABLE execution ALTER COLUMN vendor_type type varchar(64);
|
||||||
|
ALTER TABLE schedule ALTER COLUMN vendor_type type varchar(64);
|
||||||
|
ALTER TABLE schedule ADD COLUMN IF NOT EXISTS extra_attrs JSON;
|
||||||
|
ALTER TABLE task ALTER COLUMN vendor_type type varchar(64);
|
||||||
|
27
src/controller/gc/callback.go
Normal file
27
src/controller/gc/callback.go
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
package gc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"github.com/goharbor/harbor/src/lib/log"
|
||||||
|
"github.com/goharbor/harbor/src/lib/orm"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/task"
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
err := scheduler.RegisterCallbackFunc(SchedulerCallback, gcCallback)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("failed to registry GC call back, %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func gcCallback(ctx context.Context, p string) error {
|
||||||
|
param := &Policy{}
|
||||||
|
if err := json.Unmarshal([]byte(p), param); err != nil {
|
||||||
|
return fmt.Errorf("failed to unmarshal the param: %v", err)
|
||||||
|
}
|
||||||
|
_, err := Ctl.Start(orm.Context(), *param, task.ExecutionTriggerSchedule)
|
||||||
|
return err
|
||||||
|
}
|
217
src/controller/gc/controller.go
Normal file
217
src/controller/gc/controller.go
Normal file
@ -0,0 +1,217 @@
|
|||||||
|
package gc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"github.com/goharbor/harbor/src/jobservice/job"
|
||||||
|
"github.com/goharbor/harbor/src/lib/errors"
|
||||||
|
"github.com/goharbor/harbor/src/lib/q"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/task"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// Ctl is a global garbage collection controller instance
|
||||||
|
Ctl = NewController()
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// SchedulerCallback ...
|
||||||
|
SchedulerCallback = "GARBAGE_COLLECTION"
|
||||||
|
// GCVendorType ...
|
||||||
|
GCVendorType = "GARBAGE_COLLECTION"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Controller manages the tags
|
||||||
|
type Controller interface {
|
||||||
|
// Start start a manual gc job
|
||||||
|
Start(ctx context.Context, policy Policy, trigger string) (int64, error)
|
||||||
|
// Stop stop a gc job
|
||||||
|
Stop(ctx context.Context, id int64) error
|
||||||
|
|
||||||
|
// ExecutionCount returns the total count of executions according to the query
|
||||||
|
ExecutionCount(ctx context.Context, query *q.Query) (count int64, err error)
|
||||||
|
// ListExecutions lists the executions according to the query
|
||||||
|
ListExecutions(ctx context.Context, query *q.Query) (executions []*Execution, err error)
|
||||||
|
// GetExecution gets the specific execution
|
||||||
|
GetExecution(ctx context.Context, executionID int64) (execution *Execution, err error)
|
||||||
|
|
||||||
|
// GetTask gets the specific task
|
||||||
|
GetTask(ctx context.Context, id int64) (*Task, error)
|
||||||
|
// GetTaskLog gets log of the specific task
|
||||||
|
GetTaskLog(ctx context.Context, id int64) ([]byte, error)
|
||||||
|
|
||||||
|
// GetSchedule get the current gc schedule
|
||||||
|
GetSchedule(ctx context.Context) (*scheduler.Schedule, error)
|
||||||
|
// CreateSchedule create the gc schedule with cron type & string
|
||||||
|
CreateSchedule(ctx context.Context, cronType, cron string, policy Policy) (int64, error)
|
||||||
|
// DeleteSchedule remove the gc schedule
|
||||||
|
DeleteSchedule(ctx context.Context) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewController creates an instance of the default repository controller
|
||||||
|
func NewController() Controller {
|
||||||
|
return &controller{
|
||||||
|
taskMgr: task.NewManager(),
|
||||||
|
exeMgr: task.NewExecutionManager(),
|
||||||
|
schedulerMgr: scheduler.New(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type controller struct {
|
||||||
|
taskMgr task.Manager
|
||||||
|
exeMgr task.ExecutionManager
|
||||||
|
schedulerMgr scheduler.Scheduler
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start starts the manual GC
|
||||||
|
func (c *controller) Start(ctx context.Context, policy Policy, trigger string) (int64, error) {
|
||||||
|
para := make(map[string]interface{})
|
||||||
|
para["delete_untagged"] = policy.DeleteUntagged
|
||||||
|
para["dry_run"] = policy.DryRun
|
||||||
|
para["redis_url_reg"] = policy.ExtraAttrs["redis_url_reg"]
|
||||||
|
para["time_window"] = policy.ExtraAttrs["time_window"]
|
||||||
|
|
||||||
|
execID, err := c.exeMgr.Create(ctx, GCVendorType, -1, trigger, para)
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
_, err = c.taskMgr.Create(ctx, execID, &task.Job{
|
||||||
|
Name: job.ImageGC,
|
||||||
|
Metadata: &job.Metadata{
|
||||||
|
JobKind: job.KindGeneric,
|
||||||
|
},
|
||||||
|
Parameters: para,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
return execID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop ...
|
||||||
|
func (c *controller) Stop(ctx context.Context, id int64) error {
|
||||||
|
return c.exeMgr.Stop(ctx, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionCount ...
|
||||||
|
func (c *controller) ExecutionCount(ctx context.Context, query *q.Query) (int64, error) {
|
||||||
|
query.Keywords["VendorType"] = GCVendorType
|
||||||
|
return c.exeMgr.Count(ctx, query)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListExecutions ...
|
||||||
|
func (c *controller) ListExecutions(ctx context.Context, query *q.Query) ([]*Execution, error) {
|
||||||
|
query = q.MustClone(query)
|
||||||
|
query.Keywords["VendorType"] = GCVendorType
|
||||||
|
|
||||||
|
execs, err := c.exeMgr.List(ctx, query)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var executions []*Execution
|
||||||
|
for _, exec := range execs {
|
||||||
|
executions = append(executions, convertExecution(exec))
|
||||||
|
}
|
||||||
|
return executions, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetExecution ...
|
||||||
|
func (c *controller) GetExecution(ctx context.Context, id int64) (*Execution, error) {
|
||||||
|
execs, err := c.exeMgr.List(ctx, &q.Query{
|
||||||
|
Keywords: map[string]interface{}{
|
||||||
|
"ID": id,
|
||||||
|
"VendorType": GCVendorType,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(execs) == 0 {
|
||||||
|
return nil, errors.New(nil).WithCode(errors.NotFoundCode).
|
||||||
|
WithMessage("garbage collection execution %d not found", id)
|
||||||
|
}
|
||||||
|
return convertExecution(execs[0]), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTask ...
|
||||||
|
func (c *controller) GetTask(ctx context.Context, id int64) (*Task, error) {
|
||||||
|
tasks, err := c.taskMgr.List(ctx, &q.Query{
|
||||||
|
Keywords: map[string]interface{}{
|
||||||
|
"ID": id,
|
||||||
|
"VendorType": GCVendorType,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(tasks) == 0 {
|
||||||
|
return nil, errors.New(nil).WithCode(errors.NotFoundCode).
|
||||||
|
WithMessage("garbage collection task %d not found", id)
|
||||||
|
}
|
||||||
|
return convertTask(tasks[0]), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTaskLog ...
|
||||||
|
func (c *controller) GetTaskLog(ctx context.Context, id int64) ([]byte, error) {
|
||||||
|
_, err := c.GetTask(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return c.taskMgr.GetLog(ctx, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetSchedule ...
|
||||||
|
func (c *controller) GetSchedule(ctx context.Context) (*scheduler.Schedule, error) {
|
||||||
|
sch, err := c.schedulerMgr.ListSchedules(ctx, q.New(q.KeyWords{"VendorType": GCVendorType}))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(sch) == 0 {
|
||||||
|
return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc schedule is found")
|
||||||
|
}
|
||||||
|
if sch[0] == nil {
|
||||||
|
return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc schedule is found")
|
||||||
|
}
|
||||||
|
return sch[0], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CreateSchedule ...
|
||||||
|
func (c *controller) CreateSchedule(ctx context.Context, cronType, cron string, policy Policy) (int64, error) {
|
||||||
|
extras := make(map[string]interface{})
|
||||||
|
extras["delete_untagged"] = policy.DeleteUntagged
|
||||||
|
return c.schedulerMgr.Schedule(ctx, GCVendorType, -1, cronType, cron, SchedulerCallback, policy, extras)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteSchedule ...
|
||||||
|
func (c *controller) DeleteSchedule(ctx context.Context) error {
|
||||||
|
return c.schedulerMgr.UnScheduleByVendor(ctx, GCVendorType, -1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func convertExecution(exec *task.Execution) *Execution {
|
||||||
|
return &Execution{
|
||||||
|
ID: exec.ID,
|
||||||
|
Status: exec.Status,
|
||||||
|
StatusMessage: exec.StatusMessage,
|
||||||
|
Trigger: exec.Trigger,
|
||||||
|
ExtraAttrs: exec.ExtraAttrs,
|
||||||
|
StartTime: exec.StartTime,
|
||||||
|
EndTime: exec.EndTime,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func convertTask(task *task.Task) *Task {
|
||||||
|
return &Task{
|
||||||
|
ID: task.ID,
|
||||||
|
ExecutionID: task.ExecutionID,
|
||||||
|
Status: task.Status,
|
||||||
|
StatusMessage: task.StatusMessage,
|
||||||
|
RunCount: task.RunCount,
|
||||||
|
DeleteUntagged: task.GetBoolFromExtraAttrs("delete_untagged"),
|
||||||
|
DryRun: task.GetBoolFromExtraAttrs("dry_run"),
|
||||||
|
JobID: task.JobID,
|
||||||
|
CreationTime: task.CreationTime,
|
||||||
|
StartTime: task.StartTime,
|
||||||
|
UpdateTime: task.UpdateTime,
|
||||||
|
EndTime: task.EndTime,
|
||||||
|
}
|
||||||
|
}
|
148
src/controller/gc/controller_test.go
Normal file
148
src/controller/gc/controller_test.go
Normal file
@ -0,0 +1,148 @@
|
|||||||
|
package gc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/goharbor/harbor/src/jobservice/job"
|
||||||
|
"github.com/goharbor/harbor/src/lib/q"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/task"
|
||||||
|
"github.com/goharbor/harbor/src/testing/mock"
|
||||||
|
schedulertesting "github.com/goharbor/harbor/src/testing/pkg/scheduler"
|
||||||
|
tasktesting "github.com/goharbor/harbor/src/testing/pkg/task"
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type gcCtrTestSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
scheduler *schedulertesting.Scheduler
|
||||||
|
execMgr *tasktesting.ExecutionManager
|
||||||
|
taskMgr *tasktesting.Manager
|
||||||
|
ctl *controller
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcCtrTestSuite) SetupTest() {
|
||||||
|
g.execMgr = &tasktesting.ExecutionManager{}
|
||||||
|
g.taskMgr = &tasktesting.Manager{}
|
||||||
|
g.scheduler = &schedulertesting.Scheduler{}
|
||||||
|
g.ctl = &controller{
|
||||||
|
taskMgr: g.taskMgr,
|
||||||
|
exeMgr: g.execMgr,
|
||||||
|
schedulerMgr: g.scheduler,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcCtrTestSuite) TestStart() {
|
||||||
|
g.execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||||
|
g.taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||||
|
g.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
|
||||||
|
|
||||||
|
dataMap := make(map[string]interface{})
|
||||||
|
p := Policy{
|
||||||
|
DeleteUntagged: true,
|
||||||
|
ExtraAttrs: dataMap,
|
||||||
|
}
|
||||||
|
id, err := g.ctl.Start(nil, p, task.ExecutionTriggerManual)
|
||||||
|
g.Nil(err)
|
||||||
|
g.Equal(int64(1), id)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcCtrTestSuite) TestStop() {
|
||||||
|
g.execMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
|
||||||
|
g.Nil(g.ctl.Stop(nil, 1))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcCtrTestSuite) TestGetTaskLog() {
|
||||||
|
g.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
|
||||||
|
{
|
||||||
|
ID: 1,
|
||||||
|
ExecutionID: 1,
|
||||||
|
Status: job.SuccessStatus.String(),
|
||||||
|
},
|
||||||
|
}, nil)
|
||||||
|
g.taskMgr.On("GetLog", mock.Anything, mock.Anything).Return([]byte("hello world"), nil)
|
||||||
|
|
||||||
|
log, err := g.ctl.GetTaskLog(nil, 1)
|
||||||
|
g.Nil(err)
|
||||||
|
g.Equal([]byte("hello world"), log)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcCtrTestSuite) TestExecutionCount() {
|
||||||
|
g.execMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||||
|
count, err := g.ctl.ExecutionCount(nil, q.New(q.KeyWords{"VendorType": "gc"}))
|
||||||
|
g.Nil(err)
|
||||||
|
g.Equal(int64(1), count)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcCtrTestSuite) TestGetExecution() {
|
||||||
|
g.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
|
||||||
|
{
|
||||||
|
ID: 1,
|
||||||
|
Trigger: "Manual",
|
||||||
|
VendorType: GCVendorType,
|
||||||
|
StatusMessage: "Success",
|
||||||
|
},
|
||||||
|
}, nil)
|
||||||
|
|
||||||
|
hs, err := g.ctl.GetExecution(nil, int64(1))
|
||||||
|
g.Nil(err)
|
||||||
|
|
||||||
|
g.Equal("Manual", hs.Trigger)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcCtrTestSuite) TestListExecutions() {
|
||||||
|
g.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
|
||||||
|
{
|
||||||
|
ID: 1,
|
||||||
|
Trigger: "Manual",
|
||||||
|
},
|
||||||
|
}, nil)
|
||||||
|
|
||||||
|
g.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
|
||||||
|
{
|
||||||
|
ID: 112,
|
||||||
|
ExecutionID: 1,
|
||||||
|
Status: job.SuccessStatus.String(),
|
||||||
|
},
|
||||||
|
}, nil)
|
||||||
|
|
||||||
|
hs, err := g.ctl.ListExecutions(nil, q.New(q.KeyWords{"VendorType": "gc"}))
|
||||||
|
|
||||||
|
g.Nil(err)
|
||||||
|
g.Equal("Manual", hs[0].Trigger)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcCtrTestSuite) TestGetSchedule() {
|
||||||
|
g.scheduler.On("ListSchedules", mock.Anything, mock.Anything).Return([]*scheduler.Schedule{
|
||||||
|
{
|
||||||
|
ID: 1,
|
||||||
|
VendorType: "gc",
|
||||||
|
},
|
||||||
|
}, nil)
|
||||||
|
|
||||||
|
sche, err := g.ctl.GetSchedule(nil)
|
||||||
|
g.Nil(err)
|
||||||
|
g.Equal("gc", sche.VendorType)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcCtrTestSuite) TestCreateSchedule() {
|
||||||
|
g.scheduler.On("Schedule", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
|
||||||
|
mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||||
|
|
||||||
|
dataMap := make(map[string]interface{})
|
||||||
|
p := Policy{
|
||||||
|
DeleteUntagged: true,
|
||||||
|
ExtraAttrs: dataMap,
|
||||||
|
}
|
||||||
|
id, err := g.ctl.CreateSchedule(nil, "Daily", "* * * * * *", p)
|
||||||
|
g.Nil(err)
|
||||||
|
g.Equal(int64(1), id)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcCtrTestSuite) TestDeleteSchedule() {
|
||||||
|
g.scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||||
|
g.Nil(g.ctl.DeleteSchedule(nil))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestControllerTestSuite(t *testing.T) {
|
||||||
|
suite.Run(t, &gcCtrTestSuite{})
|
||||||
|
}
|
54
src/controller/gc/model.go
Normal file
54
src/controller/gc/model.go
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
package gc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Policy ...
|
||||||
|
type Policy struct {
|
||||||
|
Trigger *Trigger `json:"trigger"`
|
||||||
|
DeleteUntagged bool `json:"bool"`
|
||||||
|
DryRun bool `json:"dryrun"`
|
||||||
|
ExtraAttrs map[string]interface{} `json:"extra_attrs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TriggerType represents the type of trigger.
|
||||||
|
type TriggerType string
|
||||||
|
|
||||||
|
// Trigger holds info for a trigger
|
||||||
|
type Trigger struct {
|
||||||
|
Type TriggerType `json:"type"`
|
||||||
|
Settings *TriggerSettings `json:"trigger_settings"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TriggerSettings is the setting about the trigger
|
||||||
|
type TriggerSettings struct {
|
||||||
|
Cron string `json:"cron"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execution model for replication
|
||||||
|
type Execution struct {
|
||||||
|
ID int64
|
||||||
|
Status string
|
||||||
|
StatusMessage string
|
||||||
|
Trigger string
|
||||||
|
ExtraAttrs map[string]interface{}
|
||||||
|
StartTime time.Time
|
||||||
|
EndTime time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// Task model for replication
|
||||||
|
type Task struct {
|
||||||
|
ID int64
|
||||||
|
ExecutionID int64
|
||||||
|
Status string
|
||||||
|
StatusMessage string
|
||||||
|
RunCount int32
|
||||||
|
DeleteUntagged bool
|
||||||
|
DryRun bool
|
||||||
|
JobID string
|
||||||
|
CreationTime time.Time
|
||||||
|
StartTime time.Time
|
||||||
|
UpdateTime time.Time
|
||||||
|
EndTime time.Time
|
||||||
|
}
|
@ -292,8 +292,9 @@ func (c *controller) CreatePolicy(ctx context.Context, schema *policyModels.Sche
|
|||||||
schema.Trigger.Type == policyModels.TriggerTypeScheduled &&
|
schema.Trigger.Type == policyModels.TriggerTypeScheduled &&
|
||||||
len(schema.Trigger.Settings.Cron) > 0 {
|
len(schema.Trigger.Settings.Cron) > 0 {
|
||||||
// schedule and update policy
|
// schedule and update policy
|
||||||
|
extras := make(map[string]interface{})
|
||||||
if _, err = c.scheduler.Schedule(ctx, job.P2PPreheat, id, "", schema.Trigger.Settings.Cron,
|
if _, err = c.scheduler.Schedule(ctx, job.P2PPreheat, id, "", schema.Trigger.Settings.Cron,
|
||||||
SchedulerCallback, TriggerParam{PolicyID: id}); err != nil {
|
SchedulerCallback, TriggerParam{PolicyID: id}, extras); err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -384,8 +385,9 @@ func (c *controller) UpdatePolicy(ctx context.Context, schema *policyModels.Sche
|
|||||||
|
|
||||||
// schedule new
|
// schedule new
|
||||||
if needSch {
|
if needSch {
|
||||||
|
extras := make(map[string]interface{})
|
||||||
if _, err := c.scheduler.Schedule(ctx, job.P2PPreheat, schema.ID, "", cron, SchedulerCallback,
|
if _, err := c.scheduler.Schedule(ctx, job.P2PPreheat, schema.ID, "", cron, SchedulerCallback,
|
||||||
TriggerParam{PolicyID: schema.ID}); err != nil {
|
TriggerParam{PolicyID: schema.ID}, extras); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -241,7 +241,7 @@ func (s *preheatSuite) TestCreatePolicy() {
|
|||||||
FiltersStr: `[{"type":"repository","value":"harbor*"},{"type":"tag","value":"2*"}]`,
|
FiltersStr: `[{"type":"repository","value":"harbor*"},{"type":"tag","value":"2*"}]`,
|
||||||
TriggerStr: fmt.Sprintf(`{"type":"%s", "trigger_setting":{"cron":"* * * * */1"}}`, policy.TriggerTypeScheduled),
|
TriggerStr: fmt.Sprintf(`{"type":"%s", "trigger_setting":{"cron":"* * * * */1"}}`, policy.TriggerTypeScheduled),
|
||||||
}
|
}
|
||||||
s.fakeScheduler.On("Schedule", s.ctx, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
s.fakeScheduler.On("Schedule", s.ctx, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||||
s.fakePolicyMgr.On("Create", s.ctx, policy).Return(int64(1), nil)
|
s.fakePolicyMgr.On("Create", s.ctx, policy).Return(int64(1), nil)
|
||||||
s.fakePolicyMgr.On("Update", s.ctx, mock.Anything, mock.Anything).Return(nil)
|
s.fakePolicyMgr.On("Update", s.ctx, mock.Anything, mock.Anything).Return(nil)
|
||||||
s.fakeScheduler.On("UnScheduleByVendor", s.ctx, mock.Anything, mock.Anything).Return(nil)
|
s.fakeScheduler.On("UnScheduleByVendor", s.ctx, mock.Anything, mock.Anything).Return(nil)
|
||||||
|
@ -122,9 +122,6 @@ func init() {
|
|||||||
beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping")
|
beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping")
|
||||||
beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List")
|
beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List")
|
||||||
beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete")
|
beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete")
|
||||||
beego.Router("/api/system/gc/:id", &GCAPI{}, "get:GetGC")
|
|
||||||
beego.Router("/api/system/gc/:id([0-9]+)/log", &GCAPI{}, "get:GetLog")
|
|
||||||
beego.Router("/api/system/gc/schedule", &GCAPI{}, "get:Get;put:Put;post:Post")
|
|
||||||
beego.Router("/api/system/scanAll/schedule", &ScanAllAPI{}, "get:Get;put:Put;post:Post")
|
beego.Router("/api/system/scanAll/schedule", &ScanAllAPI{}, "get:Get;put:Put;post:Post")
|
||||||
beego.Router("/api/system/CVEAllowlist", &SysCVEAllowlistAPI{}, "get:Get;put:Put")
|
beego.Router("/api/system/CVEAllowlist", &SysCVEAllowlistAPI{}, "get:Get;put:Put")
|
||||||
beego.Router("/api/system/oidc/ping", &OIDCAPI{}, "post:Ping")
|
beego.Router("/api/system/oidc/ping", &OIDCAPI{}, "post:Ping")
|
||||||
@ -864,36 +861,6 @@ func (a testapi) DeleteMeta(authInfor usrInfo, projectID int64, name string) (in
|
|||||||
return code, string(body), err
|
return code, string(body), err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a testapi) AddGC(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) {
|
|
||||||
_sling := sling.New().Post(a.basePath)
|
|
||||||
|
|
||||||
path := "/api/system/gc/schedule"
|
|
||||||
|
|
||||||
_sling = _sling.Path(path)
|
|
||||||
|
|
||||||
// body params
|
|
||||||
_sling = _sling.BodyJSON(adminReq)
|
|
||||||
var httpStatusCode int
|
|
||||||
var err error
|
|
||||||
|
|
||||||
httpStatusCode, _, err = request(_sling, jsonAcceptHeader, authInfor)
|
|
||||||
|
|
||||||
return httpStatusCode, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a testapi) GCScheduleGet(authInfo usrInfo) (int, api_models.AdminJobSchedule, error) {
|
|
||||||
_sling := sling.New().Get(a.basePath)
|
|
||||||
path := "/api/system/gc/schedule"
|
|
||||||
_sling = _sling.Path(path)
|
|
||||||
httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo)
|
|
||||||
var successPayLoad api_models.AdminJobSchedule
|
|
||||||
if 200 == httpStatusCode && nil == err {
|
|
||||||
err = json.Unmarshal(body, &successPayLoad)
|
|
||||||
}
|
|
||||||
|
|
||||||
return httpStatusCode, successPayLoad, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a testapi) AddScanAll(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) {
|
func (a testapi) AddScanAll(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) {
|
||||||
_sling := sling.New().Post(a.basePath)
|
_sling := sling.New().Post(a.basePath)
|
||||||
|
|
||||||
|
@ -1,147 +0,0 @@
|
|||||||
// Copyright 2018 Project Harbor Authors
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"github.com/goharbor/harbor/src/core/config"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"strconv"
|
|
||||||
|
|
||||||
common_job "github.com/goharbor/harbor/src/common/job"
|
|
||||||
"github.com/goharbor/harbor/src/core/api/models"
|
|
||||||
)
|
|
||||||
|
|
||||||
// GCAPI handles request of harbor GC...
|
|
||||||
type GCAPI struct {
|
|
||||||
AJAPI
|
|
||||||
}
|
|
||||||
|
|
||||||
// Prepare validates the URL and parms, it needs the system admin permission.
|
|
||||||
func (gc *GCAPI) Prepare() {
|
|
||||||
gc.BaseController.Prepare()
|
|
||||||
if !gc.SecurityCtx.IsAuthenticated() {
|
|
||||||
gc.SendUnAuthorizedError(errors.New("UnAuthorized"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !gc.SecurityCtx.IsSysAdmin() {
|
|
||||||
gc.SendForbiddenError(errors.New(gc.SecurityCtx.GetUsername()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Post according to the request, it creates a cron schedule or a manual trigger for GC.
|
|
||||||
// create a daily schedule for GC
|
|
||||||
// {
|
|
||||||
// "schedule": {
|
|
||||||
// "type": "Daily",
|
|
||||||
// "cron": "0 0 0 * * *"
|
|
||||||
// },
|
|
||||||
// "parameters": {
|
|
||||||
// "delete_untagged": true
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// create a manual trigger for GC
|
|
||||||
// {
|
|
||||||
// "schedule": {
|
|
||||||
// "type": "Manual"
|
|
||||||
// },
|
|
||||||
// "parameters": {
|
|
||||||
// "delete_untagged": true
|
|
||||||
// "read_only": true
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
func (gc *GCAPI) Post() {
|
|
||||||
parameters := make(map[string]interface{})
|
|
||||||
ajr := models.AdminJobReq{
|
|
||||||
Parameters: parameters,
|
|
||||||
}
|
|
||||||
isValid, err := gc.DecodeJSONReqAndValidate(&ajr)
|
|
||||||
if !isValid {
|
|
||||||
gc.SendBadRequestError(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ajr.Parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
|
|
||||||
// default is the non-blocking GC job.
|
|
||||||
ajr.Name = common_job.ImageGC
|
|
||||||
ajr.Parameters["time_window"] = config.GetGCTimeWindow()
|
|
||||||
// if specify read_only:true, API will submit the readonly GC job, otherwise default is non-blocking GC job.
|
|
||||||
readOnlyParam, exist := ajr.Parameters["read_only"]
|
|
||||||
if exist {
|
|
||||||
if readOnly, ok := readOnlyParam.(bool); ok && readOnly {
|
|
||||||
ajr.Name = common_job.ImageGCReadOnly
|
|
||||||
}
|
|
||||||
}
|
|
||||||
gc.submit(&ajr)
|
|
||||||
gc.Redirect(http.StatusCreated, strconv.FormatInt(ajr.ID, 10))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Put handles GC cron schedule update/delete.
|
|
||||||
// Request: delete the schedule of GC
|
|
||||||
// {
|
|
||||||
// "schedule": {
|
|
||||||
// "type": "None",
|
|
||||||
// "cron": ""
|
|
||||||
// },
|
|
||||||
// "parameters": {
|
|
||||||
// "delete_untagged": true
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
func (gc *GCAPI) Put() {
|
|
||||||
parameters := make(map[string]interface{})
|
|
||||||
ajr := models.AdminJobReq{
|
|
||||||
Parameters: parameters,
|
|
||||||
}
|
|
||||||
isValid, err := gc.DecodeJSONReqAndValidate(&ajr)
|
|
||||||
if !isValid {
|
|
||||||
gc.SendBadRequestError(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ajr.Name = common_job.ImageGC
|
|
||||||
ajr.Parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
|
|
||||||
ajr.Parameters["time_window"] = config.GetGCTimeWindow()
|
|
||||||
gc.updateSchedule(ajr)
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetGC ...
|
|
||||||
func (gc *GCAPI) GetGC() {
|
|
||||||
id, err := gc.GetInt64FromPath(":id")
|
|
||||||
if err != nil {
|
|
||||||
gc.SendInternalServerError(errors.New("need to specify gc id"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
gc.get(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
// List returns the top 10 executions of GC which includes manual and cron.
|
|
||||||
func (gc *GCAPI) List() {
|
|
||||||
gc.list(common_job.ImageGC)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get gets GC schedule ...
|
|
||||||
func (gc *GCAPI) Get() {
|
|
||||||
gc.getSchedule(common_job.ImageGC)
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetLog ...
|
|
||||||
func (gc *GCAPI) GetLog() {
|
|
||||||
id, err := gc.GetInt64FromPath(":id")
|
|
||||||
if err != nil {
|
|
||||||
gc.SendBadRequestError(errors.New("invalid ID"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
gc.getLog(id)
|
|
||||||
}
|
|
@ -1,39 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/testing/apitests/apilib"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestGCPost(t *testing.T) {
|
|
||||||
|
|
||||||
adminJob001 := apilib.AdminJobReq{
|
|
||||||
Parameters: map[string]interface{}{"delete_untagged": false},
|
|
||||||
}
|
|
||||||
assert := assert.New(t)
|
|
||||||
apiTest := newHarborAPI()
|
|
||||||
|
|
||||||
// case 1: add a new admin job
|
|
||||||
code, err := apiTest.AddGC(*admin, adminJob001)
|
|
||||||
if err != nil {
|
|
||||||
t.Error("Error occurred while add a admin job", err.Error())
|
|
||||||
t.Log(err)
|
|
||||||
} else {
|
|
||||||
assert.Equal(201, code, "Add adminjob status should be 201")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGCGet(t *testing.T) {
|
|
||||||
assert := assert.New(t)
|
|
||||||
apiTest := newHarborAPI()
|
|
||||||
|
|
||||||
code, _, err := apiTest.GCScheduleGet(*admin)
|
|
||||||
if err != nil {
|
|
||||||
t.Error("Error occurred while get a admin job", err.Error())
|
|
||||||
t.Log(err)
|
|
||||||
} else {
|
|
||||||
assert.Equal(200, code, "Get adminjob status should be 200")
|
|
||||||
}
|
|
||||||
}
|
|
@ -93,10 +93,11 @@ func (r *DefaultAPIController) CreateRetention(p *policy.Metadata) (int64, error
|
|||||||
if p.Trigger.Kind == policy.TriggerKindSchedule {
|
if p.Trigger.Kind == policy.TriggerKindSchedule {
|
||||||
cron, ok := p.Trigger.Settings[policy.TriggerSettingsCron]
|
cron, ok := p.Trigger.Settings[policy.TriggerSettingsCron]
|
||||||
if ok && len(cron.(string)) > 0 {
|
if ok && len(cron.(string)) > 0 {
|
||||||
|
extras := make(map[string]interface{})
|
||||||
if _, err = r.scheduler.Schedule(orm.Context(), schedulerVendorType, id, "", cron.(string), SchedulerCallback, TriggerParam{
|
if _, err = r.scheduler.Schedule(orm.Context(), schedulerVendorType, id, "", cron.(string), SchedulerCallback, TriggerParam{
|
||||||
PolicyID: id,
|
PolicyID: id,
|
||||||
Trigger: ExecutionTriggerSchedule,
|
Trigger: ExecutionTriggerSchedule,
|
||||||
}); err != nil {
|
}, extras); err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -152,10 +153,11 @@ func (r *DefaultAPIController) UpdateRetention(p *policy.Metadata) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if needSch {
|
if needSch {
|
||||||
|
extras := make(map[string]interface{})
|
||||||
_, err := r.scheduler.Schedule(orm.Context(), schedulerVendorType, p.ID, "", p.Trigger.Settings[policy.TriggerSettingsCron].(string), SchedulerCallback, TriggerParam{
|
_, err := r.scheduler.Schedule(orm.Context(), schedulerVendorType, p.ID, "", p.Trigger.Settings[policy.TriggerSettingsCron].(string), SchedulerCallback, TriggerParam{
|
||||||
PolicyID: p.ID,
|
PolicyID: p.ID,
|
||||||
Trigger: ExecutionTriggerSchedule,
|
Trigger: ExecutionTriggerSchedule,
|
||||||
})
|
}, extras)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -220,7 +220,7 @@ func (s *ControllerTestSuite) TestExecution() {
|
|||||||
type fakeRetentionScheduler struct {
|
type fakeRetentionScheduler struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeRetentionScheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}) (int64, error) {
|
func (f *fakeRetentionScheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
|
||||||
return 111, nil
|
return 111, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -34,6 +34,7 @@ type schedule struct {
|
|||||||
VendorID int64 `orm:"column(vendor_id)"`
|
VendorID int64 `orm:"column(vendor_id)"`
|
||||||
CRONType string `orm:"column(cron_type)"`
|
CRONType string `orm:"column(cron_type)"`
|
||||||
CRON string `orm:"column(cron)"`
|
CRON string `orm:"column(cron)"`
|
||||||
|
ExtraAttrs string `orm:"column(extra_attrs)"`
|
||||||
CallbackFuncName string `orm:"column(callback_func_name)"`
|
CallbackFuncName string `orm:"column(callback_func_name)"`
|
||||||
CallbackFuncParam string `orm:"column(callback_func_param)"`
|
CallbackFuncParam string `orm:"column(callback_func_param)"`
|
||||||
CreationTime time.Time `orm:"column(creation_time)"`
|
CreationTime time.Time `orm:"column(creation_time)"`
|
||||||
|
@ -45,6 +45,7 @@ func (d *daoTestSuite) SetupTest() {
|
|||||||
CRON: "0 * * * * *",
|
CRON: "0 * * * * *",
|
||||||
CallbackFuncName: "callback_func_01",
|
CallbackFuncName: "callback_func_01",
|
||||||
CallbackFuncParam: "callback_func_params",
|
CallbackFuncParam: "callback_func_params",
|
||||||
|
ExtraAttrs: `{"key":"value"}`,
|
||||||
}
|
}
|
||||||
id, err := d.dao.Create(d.ctx, schedule)
|
id, err := d.dao.Create(d.ctx, schedule)
|
||||||
d.Require().Nil(err)
|
d.Require().Nil(err)
|
||||||
@ -79,6 +80,7 @@ func (d *daoTestSuite) TestGet() {
|
|||||||
schedule, err = d.dao.Get(d.ctx, d.id)
|
schedule, err = d.dao.Get(d.ctx, d.id)
|
||||||
d.Require().Nil(err)
|
d.Require().Nil(err)
|
||||||
d.Equal(d.id, schedule.ID)
|
d.Equal(d.id, schedule.ID)
|
||||||
|
d.Equal("{\"key\":\"value\"}", schedule.ExtraAttrs)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *daoTestSuite) TestDelete() {
|
func (d *daoTestSuite) TestDelete() {
|
||||||
@ -93,7 +95,7 @@ func (d *daoTestSuite) TestUpdate() {
|
|||||||
// not found
|
// not found
|
||||||
err := d.dao.Update(d.ctx, &schedule{
|
err := d.dao.Update(d.ctx, &schedule{
|
||||||
ID: 10000,
|
ID: 10000,
|
||||||
})
|
}, "CRON")
|
||||||
d.True(errors.IsNotFoundErr(err))
|
d.True(errors.IsNotFoundErr(err))
|
||||||
|
|
||||||
// pass
|
// pass
|
||||||
|
@ -37,14 +37,15 @@ var (
|
|||||||
|
|
||||||
// Schedule describes the detail information about the created schedule
|
// Schedule describes the detail information about the created schedule
|
||||||
type Schedule struct {
|
type Schedule struct {
|
||||||
ID int64 `json:"id"`
|
ID int64 `json:"id"`
|
||||||
VendorType string `json:"vendor_type"`
|
VendorType string `json:"vendor_type"`
|
||||||
VendorID int64 `json:"vendor_id"`
|
VendorID int64 `json:"vendor_id"`
|
||||||
CRONType string `json:"cron_type"`
|
CRONType string `json:"cron_type"`
|
||||||
CRON string `json:"cron"`
|
CRON string `json:"cron"`
|
||||||
Status string `json:"status"` // status of the underlying task(jobservice job)
|
ExtraAttrs map[string]interface{} `json:"extra_attrs"`
|
||||||
CreationTime time.Time `json:"creation_time"`
|
Status string `json:"status"` // status of the underlying task(jobservice job)
|
||||||
UpdateTime time.Time `json:"update_time"`
|
CreationTime time.Time `json:"creation_time"`
|
||||||
|
UpdateTime time.Time `json:"update_time"`
|
||||||
// we can extend this model to include more information(e.g. how many times the schedule already
|
// we can extend this model to include more information(e.g. how many times the schedule already
|
||||||
// runs; when will the schedule runs next time)
|
// runs; when will the schedule runs next time)
|
||||||
}
|
}
|
||||||
@ -59,7 +60,7 @@ type Scheduler interface {
|
|||||||
// The "params" is passed to the callback function as encoded json string, so the callback
|
// The "params" is passed to the callback function as encoded json string, so the callback
|
||||||
// function must decode it before using
|
// function must decode it before using
|
||||||
Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
|
Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
|
||||||
cron string, callbackFuncName string, params interface{}) (int64, error)
|
cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error)
|
||||||
// UnScheduleByID the schedule specified by ID
|
// UnScheduleByID the schedule specified by ID
|
||||||
UnScheduleByID(ctx context.Context, id int64) error
|
UnScheduleByID(ctx context.Context, id int64) error
|
||||||
// UnScheduleByVendor the schedule specified by vendor
|
// UnScheduleByVendor the schedule specified by vendor
|
||||||
@ -94,10 +95,10 @@ type scheduler struct {
|
|||||||
// to out of control from the global transaction, and uses a new transaction that only
|
// to out of control from the global transaction, and uses a new transaction that only
|
||||||
// covers the logic inside the function
|
// covers the logic inside the function
|
||||||
func (s *scheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
|
func (s *scheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
|
||||||
cron string, callbackFuncName string, params interface{}) (int64, error) {
|
cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
|
||||||
var scheduleID int64
|
var scheduleID int64
|
||||||
f := func(ctx context.Context) error {
|
f := func(ctx context.Context) error {
|
||||||
id, err := s.schedule(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params)
|
id, err := s.schedule(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -113,7 +114,7 @@ func (s *scheduler) Schedule(ctx context.Context, vendorType string, vendorID in
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *scheduler) schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
|
func (s *scheduler) schedule(ctx context.Context, vendorType string, vendorID int64, cronType string,
|
||||||
cron string, callbackFuncName string, params interface{}) (int64, error) {
|
cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
|
||||||
if len(vendorType) == 0 {
|
if len(vendorType) == 0 {
|
||||||
return 0, fmt.Errorf("empty vendor type")
|
return 0, fmt.Errorf("empty vendor type")
|
||||||
}
|
}
|
||||||
@ -142,6 +143,13 @@ func (s *scheduler) schedule(ctx context.Context, vendorType string, vendorID in
|
|||||||
}
|
}
|
||||||
sched.CallbackFuncParam = string(paramsData)
|
sched.CallbackFuncParam = string(paramsData)
|
||||||
}
|
}
|
||||||
|
if extras != nil {
|
||||||
|
extrasData, err := json.Marshal(extras)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
sched.ExtraAttrs = string(extrasData)
|
||||||
|
}
|
||||||
// create schedule record
|
// create schedule record
|
||||||
// when checkin hook comes, the database record must exist,
|
// when checkin hook comes, the database record must exist,
|
||||||
// so the database record must be created first before submitting job
|
// so the database record must be created first before submitting job
|
||||||
@ -272,6 +280,15 @@ func (s *scheduler) convertSchedule(ctx context.Context, schedule *schedule) (*S
|
|||||||
CreationTime: schedule.CreationTime,
|
CreationTime: schedule.CreationTime,
|
||||||
UpdateTime: schedule.UpdateTime,
|
UpdateTime: schedule.UpdateTime,
|
||||||
}
|
}
|
||||||
|
if len(schedule.ExtraAttrs) > 0 {
|
||||||
|
extras := map[string]interface{}{}
|
||||||
|
if err := json.Unmarshal([]byte(schedule.ExtraAttrs), &extras); err != nil {
|
||||||
|
log.Errorf("failed to unmarshal the extra attributes of schedule %d: %v", schedule.ID, err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
schd.ExtraAttrs = extras
|
||||||
|
}
|
||||||
|
|
||||||
executions, err := s.execMgr.List(ctx, &q.Query{
|
executions, err := s.execMgr.List(ctx, &q.Query{
|
||||||
Keywords: map[string]interface{}{
|
Keywords: map[string]interface{}{
|
||||||
"VendorType": JobNameScheduler,
|
"VendorType": JobNameScheduler,
|
||||||
|
@ -52,15 +52,16 @@ func (s *schedulerTestSuite) SetupTest() {
|
|||||||
|
|
||||||
func (s *schedulerTestSuite) TestSchedule() {
|
func (s *schedulerTestSuite) TestSchedule() {
|
||||||
// empty vendor type
|
// empty vendor type
|
||||||
id, err := s.scheduler.Schedule(nil, "", 0, "", "0 * * * * *", "callback", nil)
|
extras := make(map[string]interface{})
|
||||||
|
id, err := s.scheduler.Schedule(nil, "", 0, "", "0 * * * * *", "callback", nil, extras)
|
||||||
s.NotNil(err)
|
s.NotNil(err)
|
||||||
|
|
||||||
// invalid cron
|
// invalid cron
|
||||||
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "", "callback", nil)
|
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "", "callback", nil, extras)
|
||||||
s.NotNil(err)
|
s.NotNil(err)
|
||||||
|
|
||||||
// callback function not exist
|
// callback function not exist
|
||||||
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "not-exist", nil)
|
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "not-exist", nil, extras)
|
||||||
s.NotNil(err)
|
s.NotNil(err)
|
||||||
|
|
||||||
// failed to submit to jobservice
|
// failed to submit to jobservice
|
||||||
@ -73,7 +74,7 @@ func (s *schedulerTestSuite) TestSchedule() {
|
|||||||
Status: job.ErrorStatus.String(),
|
Status: job.ErrorStatus.String(),
|
||||||
}, nil)
|
}, nil)
|
||||||
s.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
|
s.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
|
||||||
_, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param")
|
_, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param", extras)
|
||||||
s.Require().NotNil(err)
|
s.Require().NotNil(err)
|
||||||
s.dao.AssertExpectations(s.T())
|
s.dao.AssertExpectations(s.T())
|
||||||
s.execMgr.AssertExpectations(s.T())
|
s.execMgr.AssertExpectations(s.T())
|
||||||
@ -91,7 +92,7 @@ func (s *schedulerTestSuite) TestSchedule() {
|
|||||||
ExecutionID: 1,
|
ExecutionID: 1,
|
||||||
Status: job.SuccessStatus.String(),
|
Status: job.SuccessStatus.String(),
|
||||||
}, nil)
|
}, nil)
|
||||||
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param")
|
id, err = s.scheduler.Schedule(nil, "vendor", 1, "", "0 * * * * *", "callback", "param", extras)
|
||||||
s.Require().Nil(err)
|
s.Require().Nil(err)
|
||||||
s.Equal(int64(1), id)
|
s.Equal(int64(1), id)
|
||||||
s.dao.AssertExpectations(s.T())
|
s.dao.AssertExpectations(s.T())
|
||||||
|
@ -115,6 +115,22 @@ func (t *Task) GetStringFromExtraAttrs(key string) string {
|
|||||||
return str
|
return str
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetBoolFromExtraAttrs returns the bool value specified by key
|
||||||
|
func (t *Task) GetBoolFromExtraAttrs(key string) bool {
|
||||||
|
if len(t.ExtraAttrs) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
rt, exist := t.ExtraAttrs[key]
|
||||||
|
if !exist {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
b, ok := rt.(bool)
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
// Job is the model represents the requested jobservice job
|
// Job is the model represents the requested jobservice job
|
||||||
type Job struct {
|
type Job struct {
|
||||||
Name string
|
Name string
|
||||||
|
@ -18,7 +18,7 @@
|
|||||||
<clr-dg-cell>{{job.createTime | date:'medium'}}</clr-dg-cell>
|
<clr-dg-cell>{{job.createTime | date:'medium'}}</clr-dg-cell>
|
||||||
<clr-dg-cell>{{job.updateTime | date:'medium'}}</clr-dg-cell>
|
<clr-dg-cell>{{job.updateTime | date:'medium'}}</clr-dg-cell>
|
||||||
<clr-dg-cell>
|
<clr-dg-cell>
|
||||||
<a *ngIf="job.status.toLowerCase() === 'finished' || job.status.toLowerCase() === 'error'" target="_blank" [href]="getLogLink(job.id)"><clr-icon shape="list"></clr-icon></a>
|
<a *ngIf="job.status.toLowerCase() === 'success' || job.status.toLowerCase() === 'error'" target="_blank" [href]="getLogLink(job.id)"><clr-icon shape="list"></clr-icon></a>
|
||||||
</clr-dg-cell>
|
</clr-dg-cell>
|
||||||
</clr-dg-row>
|
</clr-dg-row>
|
||||||
<clr-dg-footer>
|
<clr-dg-footer>
|
||||||
|
@ -52,7 +52,8 @@ func (c *controller) Create(policy *model.Policy) (int64, error) {
|
|||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
if isScheduledTrigger(policy) {
|
if isScheduledTrigger(policy) {
|
||||||
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, id, "", policy.Trigger.Settings.Cron, CallbackFuncName, id); err != nil {
|
extras := make(map[string]interface{})
|
||||||
|
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, id, "", policy.Trigger.Settings.Cron, CallbackFuncName, id, extras); err != nil {
|
||||||
log.Errorf("failed to schedule the policy %d: %v", id, err)
|
log.Errorf("failed to schedule the policy %d: %v", id, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -84,7 +85,8 @@ func (c *controller) Update(policy *model.Policy) error {
|
|||||||
}
|
}
|
||||||
// schedule again if needed
|
// schedule again if needed
|
||||||
if isScheduledTrigger(policy) {
|
if isScheduledTrigger(policy) {
|
||||||
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, policy.ID, "", policy.Trigger.Settings.Cron, CallbackFuncName, policy.ID); err != nil {
|
extras := make(map[string]interface{})
|
||||||
|
if _, err = c.scheduler.Schedule(orm.Context(), job.Replication, policy.ID, "", policy.Trigger.Settings.Cron, CallbackFuncName, policy.ID, extras); err != nil {
|
||||||
return fmt.Errorf("failed to schedule the policy %d: %v", policy.ID, err)
|
return fmt.Errorf("failed to schedule the policy %d: %v", policy.ID, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -226,7 +226,7 @@ func TestCreate(t *testing.T) {
|
|||||||
|
|
||||||
// scheduled trigger
|
// scheduled trigger
|
||||||
scheduler.On("Schedule", mock.Anything, mock.Anything,
|
scheduler.On("Schedule", mock.Anything, mock.Anything,
|
||||||
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||||
_, err = ctl.Create(&model.Policy{
|
_, err = ctl.Create(&model.Policy{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
Trigger: &model.Trigger{
|
Trigger: &model.Trigger{
|
||||||
@ -269,7 +269,7 @@ func TestUpdate(t *testing.T) {
|
|||||||
|
|
||||||
// the trigger changed
|
// the trigger changed
|
||||||
scheduler.On("Schedule", mock.Anything, mock.Anything,
|
scheduler.On("Schedule", mock.Anything, mock.Anything,
|
||||||
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
|
||||||
scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything,
|
scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything,
|
||||||
mock.Anything).Return(nil)
|
mock.Anything).Return(nil)
|
||||||
|
|
||||||
|
198
src/server/v2.0/handler/gc.go
Normal file
198
src/server/v2.0/handler/gc.go
Normal file
@ -0,0 +1,198 @@
|
|||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"github.com/go-openapi/runtime/middleware"
|
||||||
|
"github.com/goharbor/harbor/src/controller/gc"
|
||||||
|
"github.com/goharbor/harbor/src/core/config"
|
||||||
|
"github.com/goharbor/harbor/src/lib/errors"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/task"
|
||||||
|
"github.com/goharbor/harbor/src/server/v2.0/handler/model"
|
||||||
|
"github.com/goharbor/harbor/src/server/v2.0/models"
|
||||||
|
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/gc"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type gcAPI struct {
|
||||||
|
BaseAPI
|
||||||
|
gcCtr gc.Controller
|
||||||
|
}
|
||||||
|
|
||||||
|
func newGCAPI() *gcAPI {
|
||||||
|
return &gcAPI{
|
||||||
|
gcCtr: gc.NewController(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcAPI) CreateGCSchedule(ctx context.Context, params operation.CreateGCScheduleParams) middleware.Responder {
|
||||||
|
id, err := g.kick(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters)
|
||||||
|
if err != nil {
|
||||||
|
return g.SendError(ctx, err)
|
||||||
|
}
|
||||||
|
// replace the /api/v2.0/system/gc/schedule/{id} to /api/v2.0/system/gc/{id}
|
||||||
|
lastSlashIndex := strings.LastIndex(params.HTTPRequest.URL.Path, "/")
|
||||||
|
if lastSlashIndex != -1 {
|
||||||
|
location := fmt.Sprintf("%s/%d", params.HTTPRequest.URL.Path[:lastSlashIndex], id)
|
||||||
|
return operation.NewCreateGCScheduleCreated().WithLocation(location)
|
||||||
|
}
|
||||||
|
return operation.NewCreateGCScheduleCreated()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcAPI) UpdateGCSchedule(ctx context.Context, params operation.UpdateGCScheduleParams) middleware.Responder {
|
||||||
|
_, err := g.kick(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters)
|
||||||
|
if err != nil {
|
||||||
|
return g.SendError(ctx, err)
|
||||||
|
}
|
||||||
|
return operation.NewUpdateGCScheduleOK()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcAPI) kick(ctx context.Context, scheType string, cron string, parameters map[string]interface{}) (int64, error) {
|
||||||
|
// set the required parameters for GC
|
||||||
|
parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
|
||||||
|
parameters["time_window"] = config.GetGCTimeWindow()
|
||||||
|
|
||||||
|
var err error
|
||||||
|
var id int64
|
||||||
|
switch scheType {
|
||||||
|
case ScheduleManual:
|
||||||
|
policy := gc.Policy{
|
||||||
|
ExtraAttrs: parameters,
|
||||||
|
}
|
||||||
|
if dryRun, ok := parameters["dry_run"].(bool); ok {
|
||||||
|
policy.DryRun = dryRun
|
||||||
|
}
|
||||||
|
if deleteUntagged, ok := parameters["delete_untagged"].(bool); ok {
|
||||||
|
policy.DeleteUntagged = deleteUntagged
|
||||||
|
}
|
||||||
|
id, err = g.gcCtr.Start(ctx, policy, task.ExecutionTriggerManual)
|
||||||
|
case ScheduleNone:
|
||||||
|
err = g.gcCtr.DeleteSchedule(ctx)
|
||||||
|
case ScheduleHourly, ScheduleDaily, ScheduleWeekly, ScheduleCustom:
|
||||||
|
policy := gc.Policy{
|
||||||
|
ExtraAttrs: parameters,
|
||||||
|
}
|
||||||
|
if dryRun, ok := parameters["dry_run"].(bool); ok {
|
||||||
|
policy.DryRun = dryRun
|
||||||
|
}
|
||||||
|
if deleteUntagged, ok := parameters["delete_untagged"].(bool); ok {
|
||||||
|
policy.DeleteUntagged = deleteUntagged
|
||||||
|
}
|
||||||
|
err = g.updateSchedule(ctx, scheType, cron, policy)
|
||||||
|
}
|
||||||
|
return id, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcAPI) createSchedule(ctx context.Context, cronType, cron string, policy gc.Policy) error {
|
||||||
|
if cron == "" {
|
||||||
|
return errors.New(nil).WithCode(errors.BadRequestCode).
|
||||||
|
WithMessage("empty cron string for gc schedule")
|
||||||
|
}
|
||||||
|
_, err := g.gcCtr.CreateSchedule(ctx, cronType, cron, policy)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcAPI) updateSchedule(ctx context.Context, cronType, cron string, policy gc.Policy) error {
|
||||||
|
if err := g.gcCtr.DeleteSchedule(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return g.createSchedule(ctx, cronType, cron, policy)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcAPI) GetGCSchedule(ctx context.Context, params operation.GetGCScheduleParams) middleware.Responder {
|
||||||
|
schedule, err := g.gcCtr.GetSchedule(ctx)
|
||||||
|
if errors.IsNotFoundErr(err) {
|
||||||
|
return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(&scheduler.Schedule{}).ToSwagger())
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return g.SendError(ctx, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(schedule).ToSwagger())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryParams) middleware.Responder {
|
||||||
|
query, err := g.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
|
||||||
|
if err != nil {
|
||||||
|
return g.SendError(ctx, err)
|
||||||
|
}
|
||||||
|
total, err := g.gcCtr.ExecutionCount(ctx, query)
|
||||||
|
if err != nil {
|
||||||
|
return g.SendError(ctx, err)
|
||||||
|
}
|
||||||
|
execs, err := g.gcCtr.ListExecutions(ctx, query)
|
||||||
|
if err != nil {
|
||||||
|
return g.SendError(ctx, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var hs []*model.GCHistory
|
||||||
|
for _, exec := range execs {
|
||||||
|
extraAttrsString, err := json.Marshal(exec.ExtraAttrs)
|
||||||
|
if err != nil {
|
||||||
|
return g.SendError(ctx, err)
|
||||||
|
}
|
||||||
|
hs = append(hs, &model.GCHistory{
|
||||||
|
ID: exec.ID,
|
||||||
|
Name: gc.GCVendorType,
|
||||||
|
Kind: exec.Trigger,
|
||||||
|
Parameters: string(extraAttrsString),
|
||||||
|
Schedule: &model.ScheduleParam{
|
||||||
|
Type: exec.Trigger,
|
||||||
|
},
|
||||||
|
Status: exec.Status,
|
||||||
|
CreationTime: exec.StartTime,
|
||||||
|
UpdateTime: exec.EndTime,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
var results []*models.GCHistory
|
||||||
|
for _, h := range hs {
|
||||||
|
results = append(results, h.ToSwagger())
|
||||||
|
}
|
||||||
|
|
||||||
|
return operation.NewGetGCHistoryOK().
|
||||||
|
WithXTotalCount(total).
|
||||||
|
WithLink(g.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
|
||||||
|
WithPayload(results)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcAPI) GetGC(ctx context.Context, params operation.GetGCParams) middleware.Responder {
|
||||||
|
exec, err := g.gcCtr.GetExecution(ctx, params.GcID)
|
||||||
|
if err != nil {
|
||||||
|
return g.SendError(ctx, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
extraAttrsString, err := json.Marshal(exec.ExtraAttrs)
|
||||||
|
if err != nil {
|
||||||
|
return g.SendError(ctx, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
res := &model.GCHistory{
|
||||||
|
ID: exec.ID,
|
||||||
|
Name: gc.GCVendorType,
|
||||||
|
Kind: exec.Trigger,
|
||||||
|
Parameters: string(extraAttrsString),
|
||||||
|
Status: exec.Status,
|
||||||
|
Schedule: &model.ScheduleParam{
|
||||||
|
Type: exec.Trigger,
|
||||||
|
},
|
||||||
|
CreationTime: exec.StartTime,
|
||||||
|
UpdateTime: exec.EndTime,
|
||||||
|
}
|
||||||
|
|
||||||
|
return operation.NewGetGCOK().WithPayload(res.ToSwagger())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *gcAPI) GetGCLog(ctx context.Context, params operation.GetGCLogParams) middleware.Responder {
|
||||||
|
log, err := g.gcCtr.GetTaskLog(ctx, params.GcID)
|
||||||
|
if err != nil {
|
||||||
|
return g.SendError(ctx, err)
|
||||||
|
}
|
||||||
|
return operation.NewGetGCLogOK().WithPayload(string(log))
|
||||||
|
}
|
@ -41,6 +41,7 @@ func New() http.Handler {
|
|||||||
ReplicationAPI: newReplicationAPI(),
|
ReplicationAPI: newReplicationAPI(),
|
||||||
SysteminfoAPI: newSystemInfoAPI(),
|
SysteminfoAPI: newSystemInfoAPI(),
|
||||||
PingAPI: newPingAPI(),
|
PingAPI: newPingAPI(),
|
||||||
|
GcAPI: newGCAPI(),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
|
84
src/server/v2.0/handler/model/gc.go
Normal file
84
src/server/v2.0/handler/model/gc.go
Normal file
@ -0,0 +1,84 @@
|
|||||||
|
package model
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"github.com/go-openapi/strfmt"
|
||||||
|
"github.com/goharbor/harbor/src/lib/log"
|
||||||
|
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||||
|
"github.com/goharbor/harbor/src/server/v2.0/models"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ScheduleParam defines the parameter of schedule trigger
|
||||||
|
type ScheduleParam struct {
|
||||||
|
// Daily, Weekly, Custom, Manual, None
|
||||||
|
Type string `json:"type"`
|
||||||
|
// The cron string of scheduled job
|
||||||
|
Cron string `json:"cron"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// GCHistory gc execution history
|
||||||
|
type GCHistory struct {
|
||||||
|
Schedule *ScheduleParam `json:"schedule"`
|
||||||
|
ID int64 `json:"id"`
|
||||||
|
Name string `json:"job_name"`
|
||||||
|
Kind string `json:"job_kind"`
|
||||||
|
Parameters string `json:"job_parameters"`
|
||||||
|
Status string `json:"job_status"`
|
||||||
|
UUID string `json:"-"`
|
||||||
|
Deleted bool `json:"deleted"`
|
||||||
|
CreationTime time.Time `json:"creation_time"`
|
||||||
|
UpdateTime time.Time `json:"update_time"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ToSwagger converts the history to the swagger model
|
||||||
|
func (h *GCHistory) ToSwagger() *models.GCHistory {
|
||||||
|
return &models.GCHistory{
|
||||||
|
ID: h.ID,
|
||||||
|
JobName: h.Name,
|
||||||
|
JobKind: h.Kind,
|
||||||
|
JobParameters: h.Parameters,
|
||||||
|
Deleted: h.Deleted,
|
||||||
|
JobStatus: h.Status,
|
||||||
|
Schedule: &models.ScheduleObj{
|
||||||
|
Cron: h.Schedule.Cron,
|
||||||
|
Type: h.Schedule.Type,
|
||||||
|
},
|
||||||
|
CreationTime: strfmt.DateTime(h.CreationTime),
|
||||||
|
UpdateTime: strfmt.DateTime(h.UpdateTime),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Schedule ...
|
||||||
|
type Schedule struct {
|
||||||
|
*scheduler.Schedule
|
||||||
|
}
|
||||||
|
|
||||||
|
// ToSwagger converts the schedule to the swagger model
|
||||||
|
// TODO remove the hard code when after issue https://github.com/goharbor/harbor/issues/13047 is resolved.
|
||||||
|
func (s *Schedule) ToSwagger() *models.GCHistory {
|
||||||
|
e, err := json.Marshal(s.ExtraAttrs)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &models.GCHistory{
|
||||||
|
ID: 0,
|
||||||
|
JobName: "",
|
||||||
|
JobKind: s.CRON,
|
||||||
|
JobParameters: string(e),
|
||||||
|
Deleted: false,
|
||||||
|
JobStatus: "",
|
||||||
|
Schedule: &models.ScheduleObj{
|
||||||
|
Cron: s.CRON,
|
||||||
|
Type: "Custom",
|
||||||
|
},
|
||||||
|
CreationTime: strfmt.DateTime(s.CreationTime),
|
||||||
|
UpdateTime: strfmt.DateTime(s.UpdateTime),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSchedule ...
|
||||||
|
func NewSchedule(s *scheduler.Schedule) *Schedule {
|
||||||
|
return &Schedule{Schedule: s}
|
||||||
|
}
|
@ -29,6 +29,21 @@ import (
|
|||||||
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
|
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// ScheduleHourly : 'Hourly'
|
||||||
|
ScheduleHourly = "Hourly"
|
||||||
|
// ScheduleDaily : 'Daily'
|
||||||
|
ScheduleDaily = "Daily"
|
||||||
|
// ScheduleWeekly : 'Weekly'
|
||||||
|
ScheduleWeekly = "Weekly"
|
||||||
|
// ScheduleCustom : 'Custom'
|
||||||
|
ScheduleCustom = "Custom"
|
||||||
|
// ScheduleManual : 'Manual'
|
||||||
|
ScheduleManual = "Manual"
|
||||||
|
// ScheduleNone : 'None'
|
||||||
|
ScheduleNone = "None"
|
||||||
|
)
|
||||||
|
|
||||||
func boolValue(v *bool) bool {
|
func boolValue(v *bool) bool {
|
||||||
if v != nil {
|
if v != nil {
|
||||||
return *v
|
return *v
|
||||||
|
@ -45,10 +45,6 @@ func registerLegacyRoutes() {
|
|||||||
beego.Router("/api/"+version+"/quotas", &api.QuotaAPI{}, "get:List")
|
beego.Router("/api/"+version+"/quotas", &api.QuotaAPI{}, "get:List")
|
||||||
beego.Router("/api/"+version+"/quotas/:id([0-9]+)", &api.QuotaAPI{}, "get:Get;put:Put")
|
beego.Router("/api/"+version+"/quotas/:id([0-9]+)", &api.QuotaAPI{}, "get:Get;put:Put")
|
||||||
|
|
||||||
beego.Router("/api/"+version+"/system/gc", &api.GCAPI{}, "get:List")
|
|
||||||
beego.Router("/api/"+version+"/system/gc/:id", &api.GCAPI{}, "get:GetGC")
|
|
||||||
beego.Router("/api/"+version+"/system/gc/:id([0-9]+)/log", &api.GCAPI{}, "get:GetLog")
|
|
||||||
beego.Router("/api/"+version+"/system/gc/schedule", &api.GCAPI{}, "get:Get;put:Put;post:Post")
|
|
||||||
beego.Router("/api/"+version+"/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post")
|
beego.Router("/api/"+version+"/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post")
|
||||||
beego.Router("/api/"+version+"/system/CVEAllowlist", &api.SysCVEAllowlistAPI{}, "get:Get;put:Put")
|
beego.Router("/api/"+version+"/system/CVEAllowlist", &api.SysCVEAllowlistAPI{}, "get:Get;put:Put")
|
||||||
beego.Router("/api/"+version+"/system/oidc/ping", &api.OIDCAPI{}, "post:Ping")
|
beego.Router("/api/"+version+"/system/oidc/ping", &api.OIDCAPI{}, "post:Ping")
|
||||||
|
@ -62,20 +62,20 @@ func (_m *Scheduler) ListSchedules(ctx context.Context, query *q.Query) ([]*sche
|
|||||||
return r0, r1
|
return r0, r1
|
||||||
}
|
}
|
||||||
|
|
||||||
// Schedule provides a mock function with given fields: ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params
|
// Schedule provides a mock function with given fields: ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras
|
||||||
func (_m *Scheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}) (int64, error) {
|
func (_m *Scheduler) Schedule(ctx context.Context, vendorType string, vendorID int64, cronType string, cron string, callbackFuncName string, params interface{}, extras map[string]interface{}) (int64, error) {
|
||||||
ret := _m.Called(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params)
|
ret := _m.Called(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
|
||||||
|
|
||||||
var r0 int64
|
var r0 int64
|
||||||
if rf, ok := ret.Get(0).(func(context.Context, string, int64, string, string, string, interface{}) int64); ok {
|
if rf, ok := ret.Get(0).(func(context.Context, string, int64, string, string, string, interface{}, map[string]interface{}) int64); ok {
|
||||||
r0 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params)
|
r0 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
|
||||||
} else {
|
} else {
|
||||||
r0 = ret.Get(0).(int64)
|
r0 = ret.Get(0).(int64)
|
||||||
}
|
}
|
||||||
|
|
||||||
var r1 error
|
var r1 error
|
||||||
if rf, ok := ret.Get(1).(func(context.Context, string, int64, string, string, string, interface{}) error); ok {
|
if rf, ok := ret.Get(1).(func(context.Context, string, int64, string, string, string, interface{}, map[string]interface{}) error); ok {
|
||||||
r1 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params)
|
r1 = rf(ctx, vendorType, vendorID, cronType, cron, callbackFuncName, params, extras)
|
||||||
} else {
|
} else {
|
||||||
r1 = ret.Error(1)
|
r1 = ret.Error(1)
|
||||||
}
|
}
|
||||||
|
@ -61,6 +61,7 @@ def _create_client(server, credential, debug, api_type="products"):
|
|||||||
"scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)),
|
"scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)),
|
||||||
"replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)),
|
"replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)),
|
||||||
"robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)),
|
"robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)),
|
||||||
|
"gc": v2_swagger_client.GcApi(v2_swagger_client.ApiClient(cfg)),
|
||||||
}.get(api_type,'Error: Wrong API type')
|
}.get(api_type,'Error: Wrong API type')
|
||||||
|
|
||||||
def _assert_status_code(expect_code, return_code):
|
def _assert_status_code(expect_code, return_code):
|
||||||
|
134
tests/apitests/python/library/gc.py
Normal file
134
tests/apitests/python/library/gc.py
Normal file
@ -0,0 +1,134 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
import time
|
||||||
|
import base
|
||||||
|
import re
|
||||||
|
import v2_swagger_client
|
||||||
|
from v2_swagger_client.rest import ApiException
|
||||||
|
|
||||||
|
class GC(base.Base, object):
|
||||||
|
def __init__(self):
|
||||||
|
super(GC,self).__init__(api_type = "gc")
|
||||||
|
|
||||||
|
def get_gc_history(self, expect_status_code = 200, expect_response_body = None, **kwargs):
|
||||||
|
client = self._get_client(**kwargs)
|
||||||
|
|
||||||
|
try:
|
||||||
|
data, status_code, _ = client.get_gc_history_with_http_info()
|
||||||
|
except ApiException as e:
|
||||||
|
if e.status == expect_status_code:
|
||||||
|
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
|
||||||
|
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
|
||||||
|
else:
|
||||||
|
return e.reason, e.body
|
||||||
|
else:
|
||||||
|
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
|
||||||
|
base._assert_status_code(expect_status_code, status_code)
|
||||||
|
return data
|
||||||
|
|
||||||
|
def get_gc_status_by_id(self, job_id, expect_status_code = 200, expect_response_body = None, **kwargs):
|
||||||
|
client = self._get_client(**kwargs)
|
||||||
|
|
||||||
|
try:
|
||||||
|
data, status_code, _ = client.get_gc_with_http_info(job_id)
|
||||||
|
except ApiException as e:
|
||||||
|
if e.status == expect_status_code:
|
||||||
|
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
|
||||||
|
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
|
||||||
|
else:
|
||||||
|
return e.reason, e.body
|
||||||
|
else:
|
||||||
|
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
|
||||||
|
base._assert_status_code(expect_status_code, status_code)
|
||||||
|
return data
|
||||||
|
|
||||||
|
def get_gc_log_by_id(self, job_id, expect_status_code = 200, expect_response_body = None, **kwargs):
|
||||||
|
client = self._get_client(**kwargs)
|
||||||
|
|
||||||
|
try:
|
||||||
|
data, status_code, _ = client.get_gc_log_with_http_info(job_id)
|
||||||
|
except ApiException as e:
|
||||||
|
if e.status == expect_status_code:
|
||||||
|
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
|
||||||
|
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
|
||||||
|
else:
|
||||||
|
return e.reason, e.body
|
||||||
|
else:
|
||||||
|
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
|
||||||
|
base._assert_status_code(expect_status_code, status_code)
|
||||||
|
return data
|
||||||
|
|
||||||
|
def get_gc_schedule(self, expect_status_code = 200, expect_response_body = None, **kwargs):
|
||||||
|
client = self._get_client(**kwargs)
|
||||||
|
|
||||||
|
try:
|
||||||
|
data, status_code, _ = client.get_gc_schedule_with_http_info()
|
||||||
|
except ApiException as e:
|
||||||
|
if e.status == expect_status_code:
|
||||||
|
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
|
||||||
|
raise Exception(r"Get configuration response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
|
||||||
|
else:
|
||||||
|
return e.reason, e.body
|
||||||
|
else:
|
||||||
|
raise Exception(r"Get configuration result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
|
||||||
|
base._assert_status_code(expect_status_code, status_code)
|
||||||
|
return data
|
||||||
|
|
||||||
|
def create_gc_schedule(self, schedule_type, is_delete_untagged, cron = None, expect_status_code = 201, expect_response_body = None, **kwargs):
|
||||||
|
client = self._get_client(**kwargs)
|
||||||
|
|
||||||
|
gc_parameters = {'delete_untagged':is_delete_untagged}
|
||||||
|
|
||||||
|
gc_schedule = v2_swagger_client.ScheduleObj()
|
||||||
|
gc_schedule.type = schedule_type
|
||||||
|
if cron is not None:
|
||||||
|
gc_schedule.cron = cron
|
||||||
|
|
||||||
|
gc_job = v2_swagger_client.Schedule()
|
||||||
|
gc_job.schedule = gc_schedule
|
||||||
|
gc_job.parameters = gc_parameters
|
||||||
|
|
||||||
|
try:
|
||||||
|
_, status_code, header = client.create_gc_schedule_with_http_info(gc_job)
|
||||||
|
except ApiException as e:
|
||||||
|
if e.status == expect_status_code:
|
||||||
|
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
|
||||||
|
raise Exception(r"Create GC schedule response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
|
||||||
|
else:
|
||||||
|
return e.reason, e.body
|
||||||
|
else:
|
||||||
|
raise Exception(r"Create GC schedule result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
|
||||||
|
base._assert_status_code(expect_status_code, status_code)
|
||||||
|
return base._get_id_from_header(header)
|
||||||
|
|
||||||
|
def gc_now(self, is_delete_untagged=False, **kwargs):
|
||||||
|
gc_id = self.create_gc_schedule('Manual', is_delete_untagged, **kwargs)
|
||||||
|
return gc_id
|
||||||
|
|
||||||
|
def validate_gc_job_status(self, gc_id, expected_gc_status, **kwargs):
|
||||||
|
get_gc_status_finish = False
|
||||||
|
timeout_count = 20
|
||||||
|
while timeout_count > 0:
|
||||||
|
time.sleep(5)
|
||||||
|
status = self.get_gc_status_by_id(gc_id, **kwargs)
|
||||||
|
print("GC job No: {}, status: {}".format(timeout_count, status.job_status))
|
||||||
|
if status.job_status == expected_gc_status:
|
||||||
|
get_gc_status_finish = True
|
||||||
|
break
|
||||||
|
timeout_count = timeout_count - 1
|
||||||
|
|
||||||
|
if not (get_gc_status_finish):
|
||||||
|
raise Exception("GC status is not as expected '{}' actual GC status is '{}'".format(expected_gc_status, status.job_status))
|
||||||
|
|
||||||
|
def validate_deletion_success(self, gc_id, **kwargs):
|
||||||
|
log_content = self.get_gc_log_by_id(gc_id, **kwargs)
|
||||||
|
key_message = "manifests eligible for deletion"
|
||||||
|
key_message_pos = log_content.find(key_message)
|
||||||
|
full_message = log_content[key_message_pos-30 : key_message_pos + len(key_message)]
|
||||||
|
deleted_files_count_list = re.findall(r'\s+(\d+)\s+blobs\s+and\s+\d+\s+manifests\s+eligible\s+for\s+deletion', full_message)
|
||||||
|
|
||||||
|
if len(deleted_files_count_list) != 1:
|
||||||
|
raise Exception(r"Fail to get blobs eligible for deletion in log file, failure is {}.".format(len(deleted_files_count_list)))
|
||||||
|
deleted_files_count = int(deleted_files_count_list[0])
|
||||||
|
if deleted_files_count == 0:
|
||||||
|
raise Exception(r"Get blobs eligible for deletion count is {}, while we expect more than 1.".format(deleted_files_count))
|
@ -157,38 +157,6 @@ class System(base.Base):
|
|||||||
scan_all_id = self.create_scan_all_schedule('Manual', **kwargs)
|
scan_all_id = self.create_scan_all_schedule('Manual', **kwargs)
|
||||||
return scan_all_id
|
return scan_all_id
|
||||||
|
|
||||||
def gc_now(self, is_delete_untagged=False, **kwargs):
|
|
||||||
gc_id = self.create_gc_schedule('Manual', is_delete_untagged, **kwargs)
|
|
||||||
return gc_id
|
|
||||||
|
|
||||||
def validate_gc_job_status(self, gc_id, expected_gc_status, **kwargs):
|
|
||||||
get_gc_status_finish = False
|
|
||||||
timeout_count = 20
|
|
||||||
while timeout_count > 0:
|
|
||||||
time.sleep(5)
|
|
||||||
status = self.get_gc_status_by_id(gc_id, **kwargs)
|
|
||||||
print("GC job No: {}, status: {}".format(timeout_count, status.job_status))
|
|
||||||
if status.job_status == expected_gc_status:
|
|
||||||
get_gc_status_finish = True
|
|
||||||
break
|
|
||||||
timeout_count = timeout_count - 1
|
|
||||||
|
|
||||||
if not (get_gc_status_finish):
|
|
||||||
raise Exception("GC status is not as expected '{}' actual GC status is '{}'".format(expected_gc_status, status.job_status))
|
|
||||||
|
|
||||||
def validate_deletion_success(self, gc_id, **kwargs):
|
|
||||||
log_content = self.get_gc_log_by_id(gc_id, **kwargs)
|
|
||||||
key_message = "manifests eligible for deletion"
|
|
||||||
key_message_pos = log_content.find(key_message)
|
|
||||||
full_message = log_content[key_message_pos-30 : key_message_pos + len(key_message)]
|
|
||||||
deleted_files_count_list = re.findall(r'\s+(\d+)\s+blobs\s+and\s+\d+\s+manifests\s+eligible\s+for\s+deletion', full_message)
|
|
||||||
|
|
||||||
if len(deleted_files_count_list) != 1:
|
|
||||||
raise Exception(r"Fail to get blobs eligible for deletion in log file, failure is {}.".format(len(deleted_files_count_list)))
|
|
||||||
deleted_files_count = int(deleted_files_count_list[0])
|
|
||||||
if deleted_files_count == 0:
|
|
||||||
raise Exception(r"Get blobs eligible for deletion count is {}, while we expect more than 1.".format(deleted_files_count))
|
|
||||||
|
|
||||||
def set_cve_allowlist(self, expires_at=None, expected_status_code=200, *cve_ids, **kwargs):
|
def set_cve_allowlist(self, expires_at=None, expected_status_code=200, *cve_ids, **kwargs):
|
||||||
client = self._get_client(**kwargs)
|
client = self._get_client(**kwargs)
|
||||||
cve_list = [swagger_client.CVEAllowlistItem(cve_id=c) for c in cve_ids]
|
cve_list = [swagger_client.CVEAllowlistItem(cve_id=c) for c in cve_ids]
|
||||||
|
@ -7,17 +7,17 @@ from testutils import ADMIN_CLIENT, suppress_urllib3_warning
|
|||||||
from testutils import TEARDOWN
|
from testutils import TEARDOWN
|
||||||
from testutils import harbor_server
|
from testutils import harbor_server
|
||||||
from library.user import User
|
from library.user import User
|
||||||
from library.system import System
|
|
||||||
from library.project import Project
|
from library.project import Project
|
||||||
from library.repository import Repository
|
from library.repository import Repository
|
||||||
from library.base import _assert_status_code
|
from library.base import _assert_status_code
|
||||||
from library.repository import push_special_image_to_project
|
from library.repository import push_special_image_to_project
|
||||||
from library.artifact import Artifact
|
from library.artifact import Artifact
|
||||||
|
from library.gc import GC
|
||||||
|
|
||||||
class TestProjects(unittest.TestCase):
|
class TestProjects(unittest.TestCase):
|
||||||
@suppress_urllib3_warning
|
@suppress_urllib3_warning
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
self.system = System()
|
self.gc = GC()
|
||||||
self.project = Project()
|
self.project = Project()
|
||||||
self.user = User()
|
self.user = User()
|
||||||
self.repo = Repository()
|
self.repo = Repository()
|
||||||
@ -82,13 +82,13 @@ class TestProjects(unittest.TestCase):
|
|||||||
self.artifact.delete_tag(TestProjects.project_gc_untag_name, self.repo_name_untag, self.tag, self.tag, **ADMIN_CLIENT)
|
self.artifact.delete_tag(TestProjects.project_gc_untag_name, self.repo_name_untag, self.tag, self.tag, **ADMIN_CLIENT)
|
||||||
|
|
||||||
#5. Tigger garbage collection operation;
|
#5. Tigger garbage collection operation;
|
||||||
gc_id = self.system.gc_now(**ADMIN_CLIENT)
|
gc_id = self.gc.gc_now(**ADMIN_CLIENT)
|
||||||
|
|
||||||
#6. Check garbage collection job was finished;
|
#6. Check garbage collection job was finished;
|
||||||
self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT)
|
self.gc.validate_gc_job_status(gc_id, "Success", **ADMIN_CLIENT)
|
||||||
|
|
||||||
#7. Get garbage collection log, check there is a number of files was deleted;
|
#7. Get garbage collection log, check there is a number of files was deleted;
|
||||||
self.system.validate_deletion_success(gc_id, **ADMIN_CLIENT)
|
self.gc.validate_deletion_success(gc_id, **ADMIN_CLIENT)
|
||||||
|
|
||||||
artifacts = self.artifact.list_artifacts(TestProjects.project_gc_untag_name, self.repo_name_untag, **TestProjects.USER_GC_CLIENT)
|
artifacts = self.artifact.list_artifacts(TestProjects.project_gc_untag_name, self.repo_name_untag, **TestProjects.USER_GC_CLIENT)
|
||||||
_assert_status_code(len(artifacts), 1)
|
_assert_status_code(len(artifacts), 1)
|
||||||
@ -96,13 +96,13 @@ class TestProjects(unittest.TestCase):
|
|||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
|
|
||||||
#9. Tigger garbage collection operation;
|
#9. Tigger garbage collection operation;
|
||||||
gc_id = self.system.gc_now(is_delete_untagged=True, **ADMIN_CLIENT)
|
gc_id = self.gc.gc_now(is_delete_untagged=True, **ADMIN_CLIENT)
|
||||||
|
|
||||||
#10. Check garbage collection job was finished;
|
#10. Check garbage collection job was finished;
|
||||||
self.system.validate_gc_job_status(gc_id, "finished", **ADMIN_CLIENT)
|
self.gc.validate_gc_job_status(gc_id, "Success", **ADMIN_CLIENT)
|
||||||
|
|
||||||
#7. Get garbage collection log, check there is a number of files was deleted;
|
#7. Get garbage collection log, check there is a number of files was deleted;
|
||||||
self.system.validate_deletion_success(gc_id, **ADMIN_CLIENT)
|
self.gc.validate_deletion_success(gc_id, **ADMIN_CLIENT)
|
||||||
|
|
||||||
#11. Repository with untag image should be still there;
|
#11. Repository with untag image should be still there;
|
||||||
repo_data_untag = self.repo.list_repositories(TestProjects.project_gc_untag_name, **TestProjects.USER_GC_CLIENT)
|
repo_data_untag = self.repo.list_repositories(TestProjects.project_gc_untag_name, **TestProjects.USER_GC_CLIENT)
|
||||||
|
Loading…
Reference in New Issue
Block a user