Migrate to task manager (#129)

1, remove the gc to new programming model
2, move api define to harbor v2 swagger
3, leverage task & execution manager to manage gc job schedule, trigger and log.

Signed-off-by: wang yan <wangyan@vmware.com>
This commit is contained in:
Wang Yan 2020-09-23 13:59:42 +08:00 committed by Wang Yan
parent 4acb708938
commit dba5522d0b
16 changed files with 752 additions and 339 deletions

View File

@ -1549,25 +1549,6 @@ paths:
description: Only admin has this authority.
'500':
description: Unexpected internal errors.
/system/gc:
get:
summary: Get gc results.
description: This endpoint let user get latest ten gc results.
tags:
- Products
responses:
'200':
description: Get gc results successfully.
schema:
type: array
items:
$ref: '#/definitions/GCResult'
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'500':
description: Unexpected internal errors.
'/system/gc/{id}':
get:
summary: Get gc status.
@ -1592,101 +1573,6 @@ paths:
description: User does not have permission of admin role.
'500':
description: Unexpected internal errors.
'/system/gc/{id}/log':
get:
summary: Get gc job log.
description: This endpoint let user get gc job logs filtered by specific ID.
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Relevant job ID
tags:
- Products
responses:
'200':
description: Get successfully.
schema:
type: string
'400':
description: Illegal format of provided ID value.
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'404':
description: The specific gc ID's log does not exist.
'500':
description: Unexpected internal errors.
/system/gc/schedule:
get:
summary: Get gc's schedule.
description: This endpoint is for get schedule of gc job.
tags:
- Products
responses:
'200':
description: Get gc's schedule.
schema:
$ref: '#/definitions/AdminJobSchedule'
'401':
description: User need to log in first.
'403':
description: Only admin has this authority.
'500':
description: Unexpected internal errors.
put:
summary: Update gc's schedule.
description: |
This endpoint is for update gc schedule.
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/AdminJobSchedule'
description: Updates of gc's schedule.
tags:
- Products
responses:
'200':
description: Updated gc's schedule successfully.
'400':
description: Invalid schedule type.
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'500':
description: Unexpected internal errors.
post:
summary: Create a gc schedule.
description: |
This endpoint is for update gc schedule.
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/AdminJobSchedule'
description: Updates of gc's schedule.
tags:
- Products
responses:
'200':
description: GC schedule successfully.
'400':
description: Invalid schedule type.
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'409':
description: There is a "gc" job in progress, so the request cannot be served.
'500':
description: Unexpected internal errors.
/system/scanAll/schedule:
get:
summary: Get scan_all's schedule.

View File

@ -2017,6 +2017,153 @@ paths:
description: Not found the default root certificate.
'500':
$ref: '#/responses/500'
/system/gc:
get:
summary: Get gc results.
description: This endpoint let user get gc execution history.
tags:
- gc
operationId: getGCHistory
parameters:
- $ref: '#/parameters/query'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
responses:
'200':
description: Get gc results successfully.
headers:
X-Total-Count:
description: The total count of history
type: integer
Link:
description: Link refers to the previous page and next page
type: string
schema:
type: array
items:
$ref: '#/definitions/GCHistory'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
/system/gc/{gc_id}:
get:
summary: Get gc status.
description: This endpoint let user get gc status filtered by specific ID.
operationId: getGC
parameters:
- $ref: '#/parameters/gcId'
tags:
- gc
responses:
'200':
description: Get gc results successfully.
schema:
$ref: '#/definitions/GCHistory'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
/system/gc/{gc_id}/log:
get:
summary: Get gc job log.
description: This endpoint let user get gc job logs filtered by specific ID.
operationId: getGCLog
parameters:
- $ref: '#/parameters/gcId'
tags:
- gc
responses:
'200':
description: Get successfully.
schema:
type: string
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'404':
$ref: '#/responses/404'
'500':
$ref: '#/responses/500'
/system/gc/schedule:
get:
summary: Get gc's schedule.
description: This endpoint is for get schedule of gc job.
operationId: getSchedule
tags:
- gc
responses:
'200':
description: Get gc's schedule.
schema:
$ref: '#/definitions/GCHistory'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
post:
summary: Create a gc schedule.
description: |
This endpoint is for update gc schedule.
operationId: postSchedule
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/Schedule'
description: Updates of gc's schedule.
tags:
- gc
responses:
'201':
$ref: '#/responses/201'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'409':
$ref: '#/responses/409'
'500':
$ref: '#/responses/500'
put:
summary: Update gc's schedule.
description: |
This endpoint is for update gc schedule.
operationId: putSchedule
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/Schedule'
description: Updates of gc's schedule.
tags:
- gc
responses:
'200':
description: Updated gc's schedule successfully.
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'500':
$ref: '#/responses/500'
/ping:
get:
summary: Ping Harbor to check if it's alive.
@ -2134,6 +2281,13 @@ parameters:
description: Robot ID
required: true
type: integer
gcId:
name: gc_id
in: path
description: The ID of the gc log
required: true
type: integer
format: int64
responses:
'200':
description: Success
@ -3351,3 +3505,55 @@ definitions:
description: The storage of system.
items:
$ref: '#/definitions/Storage'
GCHistory:
type: object
properties:
id:
type: integer
description: the id of gc job.
job_name:
type: string
description: the job name of gc job.
job_kind:
type: string
description: the job kind of gc job.
job_parameters:
type: string
description: the job parameters of gc job.
schedule:
$ref: '#/definitions/ScheduleObj'
job_status:
type: string
description: the status of gc job.
deleted:
type: boolean
description: if gc job was deleted.
creation_time:
type: string
format: date-time
description: the creation time of gc job.
update_time:
type: string
format: date-time
description: the update time of gc job.
Schedule:
type: object
properties:
schedule:
$ref: '#/definitions/ScheduleObj'
parameters:
type: object
description: The parameters of admin job
additionalProperties:
type: object
ScheduleObj:
type: object
properties:
type:
type: string
description: |
The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manually' and 'None'.
'Manually' means to trigger it right away and 'None' means to cancel the schedule.
cron:
type: string
description: A cron expression, a time-based job scheduler.

View File

@ -1,4 +1,4 @@
/*
/*
Fixes issue https://github.com/goharbor/harbor/issues/13317
Ensure the role_id of maintainer is 4 and the role_id of limisted guest is 5
*/
@ -268,3 +268,6 @@ BEGIN
UPDATE scanner_registration SET is_default = TRUE WHERE name = 'Trivy' AND immutable = TRUE;
END IF;
END $$;
ALTER TABLE execution ALTER COLUMN vendor_type TYPE varchar(64);
ALTER TABLE schedule ALTER COLUMN vendor_type TYPE varchar(64);

View File

@ -0,0 +1,173 @@
package gc
import (
"context"
"encoding/json"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
)
var (
// GCCtl is a global garbage collection controller instance
GCCtl = NewController()
)
const (
// SchedulerCallback ...
SchedulerCallback = "GARBAGE_COLLECTION"
// gcVendorType ...
gcVendorType = "GARBAGE_COLLECTION"
)
// Controller manages the tags
type Controller interface {
// Start start a manual gc job
Start(ctx context.Context, parameters map[string]interface{}) error
// Stop stop a gc job
Stop(ctx context.Context, taskID int64) error
// GetLog get the gc log by id
GetLog(ctx context.Context, id int64) ([]byte, error)
// History list all of gc executions
History(ctx context.Context, query *q.Query) ([]*History, error)
// Count count the gc executions
Count(ctx context.Context, query *q.Query) (int64, error)
// GetSchedule get the current gc schedule
GetSchedule(ctx context.Context) (*scheduler.Schedule, error)
// CreateSchedule create the gc schedule with cron string
CreateSchedule(ctx context.Context, cron string, parameters map[string]interface{}) (int64, error)
// DeleteSchedule remove the gc schedule
DeleteSchedule(ctx context.Context) error
}
// NewController creates an instance of the default repository controller
func NewController() Controller {
return &controller{
taskMgr: task.NewManager(),
exeMgr: task.NewExecutionManager(),
schedulerMgr: scheduler.New(),
}
}
type controller struct {
taskMgr task.Manager
exeMgr task.ExecutionManager
schedulerMgr scheduler.Scheduler
}
// Start starts the manual GC
func (c *controller) Start(ctx context.Context, parameters map[string]interface{}) error {
execID, err := c.exeMgr.Create(ctx, gcVendorType, -1, task.ExecutionTriggerManual, parameters)
if err != nil {
return err
}
taskID, err := c.taskMgr.Create(ctx, execID, &task.Job{
Name: job.ImageGC,
Metadata: &job.Metadata{
JobKind: job.KindGeneric,
},
Parameters: parameters,
})
if err != nil {
return err
}
defer func() {
if err == nil {
return
}
if err := c.taskMgr.Stop(ctx, taskID); err != nil {
log.Errorf("failed to stop the task %d: %v", taskID, err)
}
}()
return nil
}
// Stop ...
func (c *controller) Stop(ctx context.Context, taskID int64) error {
return c.taskMgr.Stop(ctx, taskID)
}
// GetLog ...
func (c *controller) GetLog(ctx context.Context, executionID int64) ([]byte, error) {
tasks, err := c.taskMgr.List(ctx, q.New(q.KeyWords{"ExecutionID": executionID}))
if err != nil {
return nil, err
}
if len(tasks) == 0 {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc task is found")
}
return c.taskMgr.GetLog(ctx, tasks[0].ID)
}
// Count ...
func (c *controller) Count(ctx context.Context, query *q.Query) (int64, error) {
query.Keywords["VendorType"] = gcVendorType
return c.exeMgr.Count(ctx, query)
}
// History ...
func (c *controller) History(ctx context.Context, query *q.Query) ([]*History, error) {
var hs []*History
query.Keywords["VendorType"] = gcVendorType
exes, err := c.exeMgr.List(ctx, query)
if err != nil {
return nil, err
}
for _, exe := range exes {
tasks, err := c.taskMgr.List(ctx, q.New(q.KeyWords{"ExecutionID": exe.ID}))
if err != nil {
return nil, err
}
if len(tasks) == 0 {
continue
}
extraAttrsString, err := json.Marshal(exe.ExtraAttrs)
if err != nil {
return nil, err
}
hs = append(hs, &History{
ID: exe.ID,
Name: gcVendorType,
Kind: exe.Trigger,
Parameters: string(extraAttrsString),
Deleted: false,
Schedule: Schedule{Schedule: &ScheduleParam{
Type: exe.Trigger,
}},
Status: tasks[0].Status,
CreationTime: tasks[0].CreationTime,
UpdateTime: tasks[0].UpdateTime,
})
}
return hs, nil
}
// GetSchedule ...
func (c *controller) GetSchedule(ctx context.Context) (*scheduler.Schedule, error) {
sch, err := c.schedulerMgr.ListSchedules(ctx, q.New(q.KeyWords{"VendorType": gcVendorType}))
if err != nil {
return nil, err
}
if len(sch) == 0 {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc schedule is found")
}
if sch[0] == nil {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc schedule is found")
}
return sch[0], nil
}
// CreateSchedule ...
func (c *controller) CreateSchedule(ctx context.Context, cron string, parameters map[string]interface{}) (int64, error) {
return c.schedulerMgr.Schedule(ctx, gcVendorType, -1, cron, SchedulerCallback, parameters)
}
// DeleteSchedule ...
func (c *controller) DeleteSchedule(ctx context.Context) error {
return c.schedulerMgr.UnScheduleByVendor(ctx, gcVendorType, -1)
}

View File

@ -0,0 +1,122 @@
package gc
import (
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
"github.com/goharbor/harbor/src/testing/mock"
schedulertesting "github.com/goharbor/harbor/src/testing/pkg/scheduler"
tasktesting "github.com/goharbor/harbor/src/testing/pkg/task"
"github.com/stretchr/testify/suite"
"testing"
)
type gcCtrTestSuite struct {
suite.Suite
scheduler *schedulertesting.Scheduler
execMgr *tasktesting.FakeExecutionManager
taskMgr *tasktesting.FakeManager
ctl *controller
}
func (g *gcCtrTestSuite) SetupTest() {
g.execMgr = &tasktesting.FakeExecutionManager{}
g.taskMgr = &tasktesting.FakeManager{}
g.scheduler = &schedulertesting.Scheduler{}
g.ctl = &controller{
taskMgr: g.taskMgr,
exeMgr: g.execMgr,
schedulerMgr: g.scheduler,
}
}
func (g *gcCtrTestSuite) TestStart() {
g.execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
g.taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
g.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
dataMap := make(map[string]interface{})
g.Nil(g.ctl.Start(nil, dataMap))
}
func (g *gcCtrTestSuite) TestStop() {
g.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil)
g.Nil(g.ctl.Stop(nil, 1))
}
func (g *gcCtrTestSuite) TestLog() {
g.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
{
ID: 1,
ExecutionID: 1,
Status: job.SuccessStatus.String(),
},
}, nil)
g.taskMgr.On("GetLog", mock.Anything, mock.Anything).Return([]byte("hello world"), nil)
log, err := g.ctl.GetLog(nil, 1)
g.Nil(err)
g.Equal([]byte("hello world"), log)
}
func (g *gcCtrTestSuite) TestCount() {
g.execMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil)
count, err := g.ctl.Count(nil, q.New(q.KeyWords{"VendorType": "gc"}))
g.Nil(err)
g.Equal(int64(1), count)
}
func (g *gcCtrTestSuite) TestHistory() {
g.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{
{
ID: 1,
Trigger: "Manual",
},
}, nil)
g.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{
{
ID: 112,
ExecutionID: 1,
Status: job.SuccessStatus.String(),
},
}, nil)
hs, err := g.ctl.History(nil, q.New(q.KeyWords{"VendorType": "gc"}))
g.Nil(err)
g.Equal("Manual", hs[0].Kind)
}
func (g *gcCtrTestSuite) TestGetSchedule() {
g.scheduler.On("ListSchedules", mock.Anything, mock.Anything).Return([]*scheduler.Schedule{
{
ID: 1,
VendorType: "gc",
},
}, nil)
sche, err := g.ctl.GetSchedule(nil)
g.Nil(err)
g.Equal("gc", sche.VendorType)
}
func (g *gcCtrTestSuite) TestCreateSchedule() {
g.scheduler.On("Schedule", mock.Anything, mock.Anything, mock.Anything,
mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
dataMap := make(map[string]interface{})
id, err := g.ctl.CreateSchedule(nil, "", dataMap)
g.Nil(err)
g.Equal(int64(1), id)
}
func (g *gcCtrTestSuite) TestDeleteSchedule() {
g.scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything, mock.Anything).Return(nil)
g.Nil(g.ctl.DeleteSchedule(nil))
}
func TestControllerTestSuite(t *testing.T) {
suite.Run(t, &gcCtrTestSuite{})
}

View File

@ -0,0 +1,32 @@
package gc
import (
"time"
)
// Schedule ...
type Schedule struct {
Schedule *ScheduleParam `json:"schedule"`
}
// ScheduleParam defines the parameter of schedule trigger
type ScheduleParam struct {
// Daily, Weekly, Custom, Manual, None
Type string `json:"type"`
// The cron string of scheduled job
Cron string `json:"cron"`
}
// History gc execution history
type History struct {
Schedule
ID int64 `json:"id"`
Name string `json:"job_name"`
Kind string `json:"job_kind"`
Parameters string `json:"job_parameters"`
Status string `json:"job_status"`
UUID string `json:"-"`
Deleted bool `json:"deleted"`
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
}

View File

@ -18,6 +18,8 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg/task"
"net/http"
"github.com/ghodss/yaml"
@ -26,6 +28,7 @@ import (
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/security"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/controller/gc"
"github.com/goharbor/harbor/src/controller/p2p/preheat"
projectcontroller "github.com/goharbor/harbor/src/controller/project"
"github.com/goharbor/harbor/src/core/config"
@ -49,6 +52,7 @@ var (
retentionMgr retention.Manager
retentionLauncher retention.Launcher
retentionController retention.APIController
gcController gc.Controller
)
// GetRetentionController returns the retention API controller
@ -194,6 +198,8 @@ func Init() error {
retentionController = retention.NewAPIController(retentionMgr, projectMgr, repository.Mgr, scheduler.Sched, retentionLauncher)
gcController = gc.NewController()
retentionCallbackFun := func(ctx context.Context, p string) error {
param := &retention.TriggerParam{}
if err := json.Unmarshal([]byte(p), param); err != nil {
@ -217,6 +223,16 @@ func Init() error {
}
err = scheduler.RegisterCallbackFunc(preheat.SchedulerCallback, p2pPreheatCallbackFun)
gcCallbackFun := func(ctx context.Context, p string) error {
param := &gc.Policy{}
if err := json.Unmarshal([]byte(p), param); err != nil {
return fmt.Errorf("failed to unmarshal the param: %v", err)
}
_, err := gcController.Start(orm.Context(), *param, task.ExecutionTriggerSchedule)
return err
}
err = scheduler.RegisterCallbackFunc(gc.SchedulerCallback, gcCallbackFun)
return err
}

View File

@ -122,9 +122,6 @@ func init() {
beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping")
beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List")
beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete")
beego.Router("/api/system/gc/:id", &GCAPI{}, "get:GetGC")
beego.Router("/api/system/gc/:id([0-9]+)/log", &GCAPI{}, "get:GetLog")
beego.Router("/api/system/gc/schedule", &GCAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/system/scanAll/schedule", &ScanAllAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/system/CVEAllowlist", &SysCVEAllowlistAPI{}, "get:Get;put:Put")
beego.Router("/api/system/oidc/ping", &OIDCAPI{}, "post:Ping")
@ -864,36 +861,6 @@ func (a testapi) DeleteMeta(authInfor usrInfo, projectID int64, name string) (in
return code, string(body), err
}
func (a testapi) AddGC(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) {
_sling := sling.New().Post(a.basePath)
path := "/api/system/gc/schedule"
_sling = _sling.Path(path)
// body params
_sling = _sling.BodyJSON(adminReq)
var httpStatusCode int
var err error
httpStatusCode, _, err = request(_sling, jsonAcceptHeader, authInfor)
return httpStatusCode, err
}
func (a testapi) GCScheduleGet(authInfo usrInfo) (int, api_models.AdminJobSchedule, error) {
_sling := sling.New().Get(a.basePath)
path := "/api/system/gc/schedule"
_sling = _sling.Path(path)
httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo)
var successPayLoad api_models.AdminJobSchedule
if 200 == httpStatusCode && nil == err {
err = json.Unmarshal(body, &successPayLoad)
}
return httpStatusCode, successPayLoad, err
}
func (a testapi) AddScanAll(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) {
_sling := sling.New().Post(a.basePath)

View File

@ -1,147 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"errors"
"github.com/goharbor/harbor/src/core/config"
"net/http"
"os"
"strconv"
common_job "github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/core/api/models"
)
// GCAPI handles request of harbor GC...
type GCAPI struct {
AJAPI
}
// Prepare validates the URL and parms, it needs the system admin permission.
func (gc *GCAPI) Prepare() {
gc.BaseController.Prepare()
if !gc.SecurityCtx.IsAuthenticated() {
gc.SendUnAuthorizedError(errors.New("UnAuthorized"))
return
}
if !gc.SecurityCtx.IsSysAdmin() {
gc.SendForbiddenError(errors.New(gc.SecurityCtx.GetUsername()))
return
}
}
// Post according to the request, it creates a cron schedule or a manual trigger for GC.
// create a daily schedule for GC
// {
// "schedule": {
// "type": "Daily",
// "cron": "0 0 0 * * *"
// },
// "parameters": {
// "delete_untagged": true
// }
// }
// create a manual trigger for GC
// {
// "schedule": {
// "type": "Manual"
// },
// "parameters": {
// "delete_untagged": true
// "read_only": true
// }
// }
func (gc *GCAPI) Post() {
parameters := make(map[string]interface{})
ajr := models.AdminJobReq{
Parameters: parameters,
}
isValid, err := gc.DecodeJSONReqAndValidate(&ajr)
if !isValid {
gc.SendBadRequestError(err)
return
}
ajr.Parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
// default is the non-blocking GC job.
ajr.Name = common_job.ImageGC
ajr.Parameters["time_window"] = config.GetGCTimeWindow()
// if specify read_only:true, API will submit the readonly GC job, otherwise default is non-blocking GC job.
readOnlyParam, exist := ajr.Parameters["read_only"]
if exist {
if readOnly, ok := readOnlyParam.(bool); ok && readOnly {
ajr.Name = common_job.ImageGCReadOnly
}
}
gc.submit(&ajr)
gc.Redirect(http.StatusCreated, strconv.FormatInt(ajr.ID, 10))
}
// Put handles GC cron schedule update/delete.
// Request: delete the schedule of GC
// {
// "schedule": {
// "type": "None",
// "cron": ""
// },
// "parameters": {
// "delete_untagged": true
// }
// }
func (gc *GCAPI) Put() {
parameters := make(map[string]interface{})
ajr := models.AdminJobReq{
Parameters: parameters,
}
isValid, err := gc.DecodeJSONReqAndValidate(&ajr)
if !isValid {
gc.SendBadRequestError(err)
return
}
ajr.Name = common_job.ImageGC
ajr.Parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
ajr.Parameters["time_window"] = config.GetGCTimeWindow()
gc.updateSchedule(ajr)
}
// GetGC ...
func (gc *GCAPI) GetGC() {
id, err := gc.GetInt64FromPath(":id")
if err != nil {
gc.SendInternalServerError(errors.New("need to specify gc id"))
return
}
gc.get(id)
}
// List returns the top 10 executions of GC which includes manual and cron.
func (gc *GCAPI) List() {
gc.list(common_job.ImageGC)
}
// Get gets GC schedule ...
func (gc *GCAPI) Get() {
gc.getSchedule(common_job.ImageGC)
}
// GetLog ...
func (gc *GCAPI) GetLog() {
id, err := gc.GetInt64FromPath(":id")
if err != nil {
gc.SendBadRequestError(errors.New("invalid ID"))
return
}
gc.getLog(id)
}

View File

@ -1,39 +0,0 @@
package api
import (
"testing"
"github.com/goharbor/harbor/src/testing/apitests/apilib"
"github.com/stretchr/testify/assert"
)
func TestGCPost(t *testing.T) {
adminJob001 := apilib.AdminJobReq{
Parameters: map[string]interface{}{"delete_untagged": false},
}
assert := assert.New(t)
apiTest := newHarborAPI()
// case 1: add a new admin job
code, err := apiTest.AddGC(*admin, adminJob001)
if err != nil {
t.Error("Error occurred while add a admin job", err.Error())
t.Log(err)
} else {
assert.Equal(201, code, "Add adminjob status should be 201")
}
}
func TestGCGet(t *testing.T) {
assert := assert.New(t)
apiTest := newHarborAPI()
code, _, err := apiTest.GCScheduleGet(*admin)
if err != nil {
t.Error("Error occurred while get a admin job", err.Error())
t.Log(err)
} else {
assert.Equal(200, code, "Get adminjob status should be 200")
}
}

View File

@ -42,6 +42,7 @@ type Schedule struct {
VendorID int64 `json:"vendor_id"`
CRONType string `json:"cron_type"`
CRON string `json:"cron"`
Param string `json:"param"`
Status string `json:"status"` // status of the underlying task(jobservice job)
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
@ -269,6 +270,7 @@ func (s *scheduler) convertSchedule(ctx context.Context, schedule *schedule) (*S
VendorID: schedule.VendorID,
CRONType: schedule.CRONType,
CRON: schedule.CRON,
Param: schedule.CallbackFuncParam,
CreationTime: schedule.CreationTime,
UpdateTime: schedule.UpdateTime,
}

View File

@ -18,7 +18,7 @@
<clr-dg-cell>{{job.createTime | date:'medium'}}</clr-dg-cell>
<clr-dg-cell>{{job.updateTime | date:'medium'}}</clr-dg-cell>
<clr-dg-cell>
<a *ngIf="job.status.toLowerCase() === 'finished' || job.status.toLowerCase() === 'error'" target="_blank" [href]="getLogLink(job.id)"><clr-icon shape="list"></clr-icon></a>
<a *ngIf="job.status.toLowerCase() === 'success' || job.status.toLowerCase() === 'error'" target="_blank" [href]="getLogLink(job.id)"><clr-icon shape="list"></clr-icon></a>
</clr-dg-cell>
</clr-dg-row>
<clr-dg-footer>

View File

@ -0,0 +1,120 @@
package handler
import (
"context"
"github.com/go-openapi/runtime/middleware"
"github.com/goharbor/harbor/src/controller/gc"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/server/v2.0/handler/model"
"github.com/goharbor/harbor/src/server/v2.0/models"
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/gc"
"os"
)
type gcAPI struct {
BaseAPI
gcCtr gc.Controller
}
func newGCAPI() *gcAPI {
return &gcAPI{
gcCtr: gc.NewController(),
}
}
func (g *gcAPI) PostSchedule(ctx context.Context, params operation.PostScheduleParams) middleware.Responder {
if err := g.parseParam(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters); err != nil {
return g.SendError(ctx, err)
}
return operation.NewPostScheduleOK()
}
func (g *gcAPI) PutSchedule(ctx context.Context, params operation.PutScheduleParams) middleware.Responder {
if err := g.parseParam(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters); err != nil {
return g.SendError(ctx, err)
}
return operation.NewPutScheduleOK()
}
func (g *gcAPI) parseParam(ctx context.Context, scheType string, cron string, parameters map[string]interface{}) error {
// set the required parameters for GC
parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG")
parameters["time_window"] = config.GetGCTimeWindow()
var err error
switch scheType {
case model.ScheduleManual:
err = g.gcCtr.Start(ctx, parameters)
case model.ScheduleNone:
err = g.gcCtr.DeleteSchedule(ctx)
case model.ScheduleHourly, model.ScheduleDaily, model.ScheduleWeekly, model.ScheduleCustom:
err = g.updateSchedule(ctx, cron, parameters)
}
return err
}
func (g *gcAPI) createSchedule(ctx context.Context, cron string, parameters map[string]interface{}) error {
if cron == "" {
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("empty cron string for gc schedule")
}
_, err := g.gcCtr.CreateSchedule(ctx, cron, parameters)
if err != nil {
return err
}
return nil
}
func (g *gcAPI) updateSchedule(ctx context.Context, cron string, parameters map[string]interface{}) error {
if err := g.gcCtr.DeleteSchedule(ctx); err != nil {
return err
}
return g.createSchedule(ctx, cron, parameters)
}
func (g *gcAPI) GetSchedule(ctx context.Context, params operation.GetScheduleParams) middleware.Responder {
schedule, err := g.gcCtr.GetSchedule(ctx)
if errors.IsNotFoundErr(err) {
return operation.NewGetScheduleOK().WithPayload(model.NewSchedule(&scheduler.Schedule{}).ToSwagger())
}
if err != nil {
return g.SendError(ctx, err)
}
return operation.NewGetScheduleOK().WithPayload(model.NewSchedule(schedule).ToSwagger())
}
func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryParams) middleware.Responder {
query, err := g.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
if err != nil {
return g.SendError(ctx, err)
}
total, err := g.gcCtr.Count(ctx, query)
if err != nil {
return g.SendError(ctx, err)
}
hs, err := g.gcCtr.History(ctx, query)
if err != nil {
return g.SendError(ctx, err)
}
var results []*models.GCHistory
for _, h := range hs {
res := &model.GCHistory{}
res.History = h
results = append(results, res.ToSwagger())
}
return operation.NewGetGCHistoryOK().
WithXTotalCount(total).
WithLink(g.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
WithPayload(results)
}
func (g *gcAPI) GetGCLog(ctx context.Context, params operation.GetGCLogParams) middleware.Responder {
log, err := g.gcCtr.GetLog(ctx, params.GcID)
if err != nil {
return g.SendError(ctx, err)
}
return operation.NewGetGCLogOK().WithPayload(string(log))
}

View File

@ -41,6 +41,7 @@ func New() http.Handler {
ReplicationAPI: newReplicationAPI(),
SysteminfoAPI: newSystemInfoAPI(),
PingAPI: newPingAPI(),
GcAPI: newGCAPI(),
})
if err != nil {
log.Fatal(err)

View File

@ -0,0 +1,75 @@
package model
import (
"github.com/go-openapi/strfmt"
"github.com/goharbor/harbor/src/controller/gc"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/server/v2.0/models"
)
const (
// ScheduleHourly : 'Hourly'
ScheduleHourly = "Hourly"
// ScheduleDaily : 'Daily'
ScheduleDaily = "Daily"
// ScheduleWeekly : 'Weekly'
ScheduleWeekly = "Weekly"
// ScheduleCustom : 'Custom'
ScheduleCustom = "Custom"
// ScheduleManual : 'Manual'
ScheduleManual = "Manual"
// ScheduleNone : 'None'
ScheduleNone = "None"
)
// GCHistory ...
type GCHistory struct {
*gc.History
}
// ToSwagger converts the history to the swagger model
func (h *GCHistory) ToSwagger() *models.GCHistory {
return &models.GCHistory{
ID: h.ID,
JobName: h.Name,
JobKind: h.Kind,
JobParameters: h.Parameters,
Deleted: h.Deleted,
JobStatus: h.Status,
Schedule: &models.ScheduleObj{
Cron: h.Schedule.Schedule.Cron,
Type: h.Schedule.Schedule.Type,
},
CreationTime: strfmt.DateTime(h.CreationTime),
UpdateTime: strfmt.DateTime(h.UpdateTime),
}
}
// Schedule ...
type Schedule struct {
*scheduler.Schedule
}
// ToSwagger converts the schedule to the swagger model
// TODO remove the hard code when after issue https://github.com/goharbor/harbor/issues/13047 is resolved.
func (s *Schedule) ToSwagger() *models.GCHistory {
return &models.GCHistory{
ID: 0,
JobName: "",
JobKind: s.CRON,
JobParameters: s.Param,
Deleted: false,
JobStatus: "",
Schedule: &models.ScheduleObj{
Cron: s.CRON,
Type: "Custom",
},
CreationTime: strfmt.DateTime(s.CreationTime),
UpdateTime: strfmt.DateTime(s.UpdateTime),
}
}
// NewSchedule ...
func NewSchedule(s *scheduler.Schedule) *Schedule {
return &Schedule{Schedule: s}
}

View File

@ -45,10 +45,6 @@ func registerLegacyRoutes() {
beego.Router("/api/"+version+"/quotas", &api.QuotaAPI{}, "get:List")
beego.Router("/api/"+version+"/quotas/:id([0-9]+)", &api.QuotaAPI{}, "get:Get;put:Put")
beego.Router("/api/"+version+"/system/gc", &api.GCAPI{}, "get:List")
beego.Router("/api/"+version+"/system/gc/:id", &api.GCAPI{}, "get:GetGC")
beego.Router("/api/"+version+"/system/gc/:id([0-9]+)/log", &api.GCAPI{}, "get:GetLog")
beego.Router("/api/"+version+"/system/gc/schedule", &api.GCAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/"+version+"/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/"+version+"/system/CVEAllowlist", &api.SysCVEAllowlistAPI{}, "get:Get;put:Put")
beego.Router("/api/"+version+"/system/oidc/ping", &api.OIDCAPI{}, "post:Ping")