From dba5522d0be3a2524a87c56badb462c8fd97dbe4 Mon Sep 17 00:00:00 2001 From: Wang Yan Date: Wed, 23 Sep 2020 13:59:42 +0800 Subject: [PATCH] Migrate to task manager (#129) 1, remove the gc to new programming model 2, move api define to harbor v2 swagger 3, leverage task & execution manager to manage gc job schedule, trigger and log. Signed-off-by: wang yan --- api/v2.0/legacy_swagger.yaml | 114 ---------- api/v2.0/swagger.yaml | 206 ++++++++++++++++++ .../postgresql/0050_2.2.0_schema.up.sql | 5 +- src/controller/gc/controller.go | 173 +++++++++++++++ src/controller/gc/controller_test.go | 122 +++++++++++ src/controller/gc/model.go | 32 +++ src/core/api/base.go | 16 ++ src/core/api/harborapi_test.go | 33 --- src/core/api/reg_gc.go | 147 ------------- src/core/api/reg_gc_test.go | 39 ---- src/pkg/scheduler/scheduler.go | 2 + .../gc/gc-history/gc-history.component.html | 2 +- src/server/v2.0/handler/gc.go | 120 ++++++++++ src/server/v2.0/handler/handler.go | 1 + src/server/v2.0/handler/model/gc.go | 75 +++++++ src/server/v2.0/route/legacy.go | 4 - 16 files changed, 752 insertions(+), 339 deletions(-) create mode 100644 src/controller/gc/controller.go create mode 100644 src/controller/gc/controller_test.go create mode 100644 src/controller/gc/model.go delete mode 100644 src/core/api/reg_gc.go delete mode 100644 src/core/api/reg_gc_test.go create mode 100644 src/server/v2.0/handler/gc.go create mode 100644 src/server/v2.0/handler/model/gc.go diff --git a/api/v2.0/legacy_swagger.yaml b/api/v2.0/legacy_swagger.yaml index 1c3542359..5e4beefe7 100644 --- a/api/v2.0/legacy_swagger.yaml +++ b/api/v2.0/legacy_swagger.yaml @@ -1549,25 +1549,6 @@ paths: description: Only admin has this authority. '500': description: Unexpected internal errors. - /system/gc: - get: - summary: Get gc results. - description: This endpoint let user get latest ten gc results. - tags: - - Products - responses: - '200': - description: Get gc results successfully. - schema: - type: array - items: - $ref: '#/definitions/GCResult' - '401': - description: User need to log in first. - '403': - description: User does not have permission of admin role. - '500': - description: Unexpected internal errors. '/system/gc/{id}': get: summary: Get gc status. @@ -1592,101 +1573,6 @@ paths: description: User does not have permission of admin role. '500': description: Unexpected internal errors. - '/system/gc/{id}/log': - get: - summary: Get gc job log. - description: This endpoint let user get gc job logs filtered by specific ID. - parameters: - - name: id - in: path - type: integer - format: int64 - required: true - description: Relevant job ID - tags: - - Products - responses: - '200': - description: Get successfully. - schema: - type: string - '400': - description: Illegal format of provided ID value. - '401': - description: User need to log in first. - '403': - description: User does not have permission of admin role. - '404': - description: The specific gc ID's log does not exist. - '500': - description: Unexpected internal errors. - /system/gc/schedule: - get: - summary: Get gc's schedule. - description: This endpoint is for get schedule of gc job. - tags: - - Products - responses: - '200': - description: Get gc's schedule. - schema: - $ref: '#/definitions/AdminJobSchedule' - '401': - description: User need to log in first. - '403': - description: Only admin has this authority. - '500': - description: Unexpected internal errors. - put: - summary: Update gc's schedule. - description: | - This endpoint is for update gc schedule. - parameters: - - name: schedule - in: body - required: true - schema: - $ref: '#/definitions/AdminJobSchedule' - description: Updates of gc's schedule. - tags: - - Products - responses: - '200': - description: Updated gc's schedule successfully. - '400': - description: Invalid schedule type. - '401': - description: User need to log in first. - '403': - description: User does not have permission of admin role. - '500': - description: Unexpected internal errors. - post: - summary: Create a gc schedule. - description: | - This endpoint is for update gc schedule. - parameters: - - name: schedule - in: body - required: true - schema: - $ref: '#/definitions/AdminJobSchedule' - description: Updates of gc's schedule. - tags: - - Products - responses: - '200': - description: GC schedule successfully. - '400': - description: Invalid schedule type. - '401': - description: User need to log in first. - '403': - description: User does not have permission of admin role. - '409': - description: There is a "gc" job in progress, so the request cannot be served. - '500': - description: Unexpected internal errors. /system/scanAll/schedule: get: summary: Get scan_all's schedule. diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index bf1785201..928713042 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -2017,6 +2017,153 @@ paths: description: Not found the default root certificate. '500': $ref: '#/responses/500' + /system/gc: + get: + summary: Get gc results. + description: This endpoint let user get gc execution history. + tags: + - gc + operationId: getGCHistory + parameters: + - $ref: '#/parameters/query' + - $ref: '#/parameters/page' + - $ref: '#/parameters/pageSize' + responses: + '200': + description: Get gc results successfully. + headers: + X-Total-Count: + description: The total count of history + type: integer + Link: + description: Link refers to the previous page and next page + type: string + schema: + type: array + items: + $ref: '#/definitions/GCHistory' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '500': + $ref: '#/responses/500' + /system/gc/{gc_id}: + get: + summary: Get gc status. + description: This endpoint let user get gc status filtered by specific ID. + operationId: getGC + parameters: + - $ref: '#/parameters/gcId' + tags: + - gc + responses: + '200': + description: Get gc results successfully. + schema: + $ref: '#/definitions/GCHistory' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' + /system/gc/{gc_id}/log: + get: + summary: Get gc job log. + description: This endpoint let user get gc job logs filtered by specific ID. + operationId: getGCLog + parameters: + - $ref: '#/parameters/gcId' + tags: + - gc + responses: + '200': + description: Get successfully. + schema: + type: string + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' + /system/gc/schedule: + get: + summary: Get gc's schedule. + description: This endpoint is for get schedule of gc job. + operationId: getSchedule + tags: + - gc + responses: + '200': + description: Get gc's schedule. + schema: + $ref: '#/definitions/GCHistory' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '500': + $ref: '#/responses/500' + post: + summary: Create a gc schedule. + description: | + This endpoint is for update gc schedule. + operationId: postSchedule + parameters: + - name: schedule + in: body + required: true + schema: + $ref: '#/definitions/Schedule' + description: Updates of gc's schedule. + tags: + - gc + responses: + '201': + $ref: '#/responses/201' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '409': + $ref: '#/responses/409' + '500': + $ref: '#/responses/500' + put: + summary: Update gc's schedule. + description: | + This endpoint is for update gc schedule. + operationId: putSchedule + parameters: + - name: schedule + in: body + required: true + schema: + $ref: '#/definitions/Schedule' + description: Updates of gc's schedule. + tags: + - gc + responses: + '200': + description: Updated gc's schedule successfully. + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '500': + $ref: '#/responses/500' /ping: get: summary: Ping Harbor to check if it's alive. @@ -2134,6 +2281,13 @@ parameters: description: Robot ID required: true type: integer + gcId: + name: gc_id + in: path + description: The ID of the gc log + required: true + type: integer + format: int64 responses: '200': description: Success @@ -3351,3 +3505,55 @@ definitions: description: The storage of system. items: $ref: '#/definitions/Storage' + GCHistory: + type: object + properties: + id: + type: integer + description: the id of gc job. + job_name: + type: string + description: the job name of gc job. + job_kind: + type: string + description: the job kind of gc job. + job_parameters: + type: string + description: the job parameters of gc job. + schedule: + $ref: '#/definitions/ScheduleObj' + job_status: + type: string + description: the status of gc job. + deleted: + type: boolean + description: if gc job was deleted. + creation_time: + type: string + format: date-time + description: the creation time of gc job. + update_time: + type: string + format: date-time + description: the update time of gc job. + Schedule: + type: object + properties: + schedule: + $ref: '#/definitions/ScheduleObj' + parameters: + type: object + description: The parameters of admin job + additionalProperties: + type: object + ScheduleObj: + type: object + properties: + type: + type: string + description: | + The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manually' and 'None'. + 'Manually' means to trigger it right away and 'None' means to cancel the schedule. + cron: + type: string + description: A cron expression, a time-based job scheduler. diff --git a/make/migrations/postgresql/0050_2.2.0_schema.up.sql b/make/migrations/postgresql/0050_2.2.0_schema.up.sql index b10273fd9..d40a48ef6 100644 --- a/make/migrations/postgresql/0050_2.2.0_schema.up.sql +++ b/make/migrations/postgresql/0050_2.2.0_schema.up.sql @@ -1,4 +1,4 @@ -/* +/* Fixes issue https://github.com/goharbor/harbor/issues/13317 Ensure the role_id of maintainer is 4 and the role_id of limisted guest is 5 */ @@ -268,3 +268,6 @@ BEGIN UPDATE scanner_registration SET is_default = TRUE WHERE name = 'Trivy' AND immutable = TRUE; END IF; END $$; +ALTER TABLE execution ALTER COLUMN vendor_type TYPE varchar(64); +ALTER TABLE schedule ALTER COLUMN vendor_type TYPE varchar(64); + diff --git a/src/controller/gc/controller.go b/src/controller/gc/controller.go new file mode 100644 index 000000000..fef9c1c6c --- /dev/null +++ b/src/controller/gc/controller.go @@ -0,0 +1,173 @@ +package gc + +import ( + "context" + "encoding/json" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/scheduler" + "github.com/goharbor/harbor/src/pkg/task" +) + +var ( + // GCCtl is a global garbage collection controller instance + GCCtl = NewController() +) + +const ( + // SchedulerCallback ... + SchedulerCallback = "GARBAGE_COLLECTION" + // gcVendorType ... + gcVendorType = "GARBAGE_COLLECTION" +) + +// Controller manages the tags +type Controller interface { + // Start start a manual gc job + Start(ctx context.Context, parameters map[string]interface{}) error + // Stop stop a gc job + Stop(ctx context.Context, taskID int64) error + // GetLog get the gc log by id + GetLog(ctx context.Context, id int64) ([]byte, error) + // History list all of gc executions + History(ctx context.Context, query *q.Query) ([]*History, error) + // Count count the gc executions + Count(ctx context.Context, query *q.Query) (int64, error) + // GetSchedule get the current gc schedule + GetSchedule(ctx context.Context) (*scheduler.Schedule, error) + // CreateSchedule create the gc schedule with cron string + CreateSchedule(ctx context.Context, cron string, parameters map[string]interface{}) (int64, error) + // DeleteSchedule remove the gc schedule + DeleteSchedule(ctx context.Context) error +} + +// NewController creates an instance of the default repository controller +func NewController() Controller { + return &controller{ + taskMgr: task.NewManager(), + exeMgr: task.NewExecutionManager(), + schedulerMgr: scheduler.New(), + } +} + +type controller struct { + taskMgr task.Manager + exeMgr task.ExecutionManager + schedulerMgr scheduler.Scheduler +} + +// Start starts the manual GC +func (c *controller) Start(ctx context.Context, parameters map[string]interface{}) error { + execID, err := c.exeMgr.Create(ctx, gcVendorType, -1, task.ExecutionTriggerManual, parameters) + if err != nil { + return err + } + taskID, err := c.taskMgr.Create(ctx, execID, &task.Job{ + Name: job.ImageGC, + Metadata: &job.Metadata{ + JobKind: job.KindGeneric, + }, + Parameters: parameters, + }) + if err != nil { + return err + } + defer func() { + if err == nil { + return + } + if err := c.taskMgr.Stop(ctx, taskID); err != nil { + log.Errorf("failed to stop the task %d: %v", taskID, err) + } + }() + return nil +} + +// Stop ... +func (c *controller) Stop(ctx context.Context, taskID int64) error { + return c.taskMgr.Stop(ctx, taskID) +} + +// GetLog ... +func (c *controller) GetLog(ctx context.Context, executionID int64) ([]byte, error) { + tasks, err := c.taskMgr.List(ctx, q.New(q.KeyWords{"ExecutionID": executionID})) + if err != nil { + return nil, err + } + if len(tasks) == 0 { + return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc task is found") + } + return c.taskMgr.GetLog(ctx, tasks[0].ID) +} + +// Count ... +func (c *controller) Count(ctx context.Context, query *q.Query) (int64, error) { + query.Keywords["VendorType"] = gcVendorType + return c.exeMgr.Count(ctx, query) +} + +// History ... +func (c *controller) History(ctx context.Context, query *q.Query) ([]*History, error) { + var hs []*History + + query.Keywords["VendorType"] = gcVendorType + exes, err := c.exeMgr.List(ctx, query) + if err != nil { + return nil, err + } + for _, exe := range exes { + tasks, err := c.taskMgr.List(ctx, q.New(q.KeyWords{"ExecutionID": exe.ID})) + if err != nil { + return nil, err + } + if len(tasks) == 0 { + continue + } + + extraAttrsString, err := json.Marshal(exe.ExtraAttrs) + if err != nil { + return nil, err + } + hs = append(hs, &History{ + ID: exe.ID, + Name: gcVendorType, + Kind: exe.Trigger, + Parameters: string(extraAttrsString), + Deleted: false, + Schedule: Schedule{Schedule: &ScheduleParam{ + Type: exe.Trigger, + }}, + Status: tasks[0].Status, + CreationTime: tasks[0].CreationTime, + UpdateTime: tasks[0].UpdateTime, + }) + } + return hs, nil +} + +// GetSchedule ... +func (c *controller) GetSchedule(ctx context.Context) (*scheduler.Schedule, error) { + sch, err := c.schedulerMgr.ListSchedules(ctx, q.New(q.KeyWords{"VendorType": gcVendorType})) + if err != nil { + return nil, err + } + if len(sch) == 0 { + return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc schedule is found") + } + if sch[0] == nil { + return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no gc schedule is found") + } + return sch[0], nil +} + +// CreateSchedule ... +func (c *controller) CreateSchedule(ctx context.Context, cron string, parameters map[string]interface{}) (int64, error) { + return c.schedulerMgr.Schedule(ctx, gcVendorType, -1, cron, SchedulerCallback, parameters) +} + +// DeleteSchedule ... +func (c *controller) DeleteSchedule(ctx context.Context) error { + return c.schedulerMgr.UnScheduleByVendor(ctx, gcVendorType, -1) +} diff --git a/src/controller/gc/controller_test.go b/src/controller/gc/controller_test.go new file mode 100644 index 000000000..3abea1c20 --- /dev/null +++ b/src/controller/gc/controller_test.go @@ -0,0 +1,122 @@ +package gc + +import ( + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/scheduler" + "github.com/goharbor/harbor/src/pkg/task" + "github.com/goharbor/harbor/src/testing/mock" + schedulertesting "github.com/goharbor/harbor/src/testing/pkg/scheduler" + tasktesting "github.com/goharbor/harbor/src/testing/pkg/task" + "github.com/stretchr/testify/suite" + "testing" +) + +type gcCtrTestSuite struct { + suite.Suite + scheduler *schedulertesting.Scheduler + execMgr *tasktesting.FakeExecutionManager + taskMgr *tasktesting.FakeManager + ctl *controller +} + +func (g *gcCtrTestSuite) SetupTest() { + g.execMgr = &tasktesting.FakeExecutionManager{} + g.taskMgr = &tasktesting.FakeManager{} + g.scheduler = &schedulertesting.Scheduler{} + g.ctl = &controller{ + taskMgr: g.taskMgr, + exeMgr: g.execMgr, + schedulerMgr: g.scheduler, + } +} + +func (g *gcCtrTestSuite) TestStart() { + g.execMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) + g.taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) + g.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil) + + dataMap := make(map[string]interface{}) + g.Nil(g.ctl.Start(nil, dataMap)) +} + +func (g *gcCtrTestSuite) TestStop() { + g.taskMgr.On("Stop", mock.Anything, mock.Anything).Return(nil) + g.Nil(g.ctl.Stop(nil, 1)) +} + +func (g *gcCtrTestSuite) TestLog() { + g.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{ + { + ID: 1, + ExecutionID: 1, + Status: job.SuccessStatus.String(), + }, + }, nil) + g.taskMgr.On("GetLog", mock.Anything, mock.Anything).Return([]byte("hello world"), nil) + + log, err := g.ctl.GetLog(nil, 1) + g.Nil(err) + g.Equal([]byte("hello world"), log) +} + +func (g *gcCtrTestSuite) TestCount() { + g.execMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil) + count, err := g.ctl.Count(nil, q.New(q.KeyWords{"VendorType": "gc"})) + g.Nil(err) + g.Equal(int64(1), count) +} + +func (g *gcCtrTestSuite) TestHistory() { + g.execMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Execution{ + { + ID: 1, + Trigger: "Manual", + }, + }, nil) + + g.taskMgr.On("List", mock.Anything, mock.Anything).Return([]*task.Task{ + { + ID: 112, + ExecutionID: 1, + Status: job.SuccessStatus.String(), + }, + }, nil) + + hs, err := g.ctl.History(nil, q.New(q.KeyWords{"VendorType": "gc"})) + + g.Nil(err) + g.Equal("Manual", hs[0].Kind) +} + +func (g *gcCtrTestSuite) TestGetSchedule() { + g.scheduler.On("ListSchedules", mock.Anything, mock.Anything).Return([]*scheduler.Schedule{ + { + ID: 1, + VendorType: "gc", + }, + }, nil) + + sche, err := g.ctl.GetSchedule(nil) + g.Nil(err) + g.Equal("gc", sche.VendorType) +} + +func (g *gcCtrTestSuite) TestCreateSchedule() { + g.scheduler.On("Schedule", mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) + + dataMap := make(map[string]interface{}) + id, err := g.ctl.CreateSchedule(nil, "", dataMap) + g.Nil(err) + g.Equal(int64(1), id) +} + +func (g *gcCtrTestSuite) TestDeleteSchedule() { + g.scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything, mock.Anything).Return(nil) + g.Nil(g.ctl.DeleteSchedule(nil)) +} + +func TestControllerTestSuite(t *testing.T) { + suite.Run(t, &gcCtrTestSuite{}) +} diff --git a/src/controller/gc/model.go b/src/controller/gc/model.go new file mode 100644 index 000000000..bd574d67e --- /dev/null +++ b/src/controller/gc/model.go @@ -0,0 +1,32 @@ +package gc + +import ( + "time" +) + +// Schedule ... +type Schedule struct { + Schedule *ScheduleParam `json:"schedule"` +} + +// ScheduleParam defines the parameter of schedule trigger +type ScheduleParam struct { + // Daily, Weekly, Custom, Manual, None + Type string `json:"type"` + // The cron string of scheduled job + Cron string `json:"cron"` +} + +// History gc execution history +type History struct { + Schedule + ID int64 `json:"id"` + Name string `json:"job_name"` + Kind string `json:"job_kind"` + Parameters string `json:"job_parameters"` + Status string `json:"job_status"` + UUID string `json:"-"` + Deleted bool `json:"deleted"` + CreationTime time.Time `json:"creation_time"` + UpdateTime time.Time `json:"update_time"` +} diff --git a/src/core/api/base.go b/src/core/api/base.go index 7356afb98..a68f1170d 100644 --- a/src/core/api/base.go +++ b/src/core/api/base.go @@ -18,6 +18,8 @@ import ( "context" "encoding/json" "fmt" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/pkg/task" "net/http" "github.com/ghodss/yaml" @@ -26,6 +28,7 @@ import ( "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/security" "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/controller/gc" "github.com/goharbor/harbor/src/controller/p2p/preheat" projectcontroller "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/core/config" @@ -49,6 +52,7 @@ var ( retentionMgr retention.Manager retentionLauncher retention.Launcher retentionController retention.APIController + gcController gc.Controller ) // GetRetentionController returns the retention API controller @@ -194,6 +198,8 @@ func Init() error { retentionController = retention.NewAPIController(retentionMgr, projectMgr, repository.Mgr, scheduler.Sched, retentionLauncher) + gcController = gc.NewController() + retentionCallbackFun := func(ctx context.Context, p string) error { param := &retention.TriggerParam{} if err := json.Unmarshal([]byte(p), param); err != nil { @@ -217,6 +223,16 @@ func Init() error { } err = scheduler.RegisterCallbackFunc(preheat.SchedulerCallback, p2pPreheatCallbackFun) + gcCallbackFun := func(ctx context.Context, p string) error { + param := &gc.Policy{} + if err := json.Unmarshal([]byte(p), param); err != nil { + return fmt.Errorf("failed to unmarshal the param: %v", err) + } + _, err := gcController.Start(orm.Context(), *param, task.ExecutionTriggerSchedule) + return err + } + err = scheduler.RegisterCallbackFunc(gc.SchedulerCallback, gcCallbackFun) + return err } diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 14ccac693..6d6460443 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -122,9 +122,6 @@ func init() { beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping") beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List") beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete") - beego.Router("/api/system/gc/:id", &GCAPI{}, "get:GetGC") - beego.Router("/api/system/gc/:id([0-9]+)/log", &GCAPI{}, "get:GetLog") - beego.Router("/api/system/gc/schedule", &GCAPI{}, "get:Get;put:Put;post:Post") beego.Router("/api/system/scanAll/schedule", &ScanAllAPI{}, "get:Get;put:Put;post:Post") beego.Router("/api/system/CVEAllowlist", &SysCVEAllowlistAPI{}, "get:Get;put:Put") beego.Router("/api/system/oidc/ping", &OIDCAPI{}, "post:Ping") @@ -864,36 +861,6 @@ func (a testapi) DeleteMeta(authInfor usrInfo, projectID int64, name string) (in return code, string(body), err } -func (a testapi) AddGC(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) { - _sling := sling.New().Post(a.basePath) - - path := "/api/system/gc/schedule" - - _sling = _sling.Path(path) - - // body params - _sling = _sling.BodyJSON(adminReq) - var httpStatusCode int - var err error - - httpStatusCode, _, err = request(_sling, jsonAcceptHeader, authInfor) - - return httpStatusCode, err -} - -func (a testapi) GCScheduleGet(authInfo usrInfo) (int, api_models.AdminJobSchedule, error) { - _sling := sling.New().Get(a.basePath) - path := "/api/system/gc/schedule" - _sling = _sling.Path(path) - httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo) - var successPayLoad api_models.AdminJobSchedule - if 200 == httpStatusCode && nil == err { - err = json.Unmarshal(body, &successPayLoad) - } - - return httpStatusCode, successPayLoad, err -} - func (a testapi) AddScanAll(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) { _sling := sling.New().Post(a.basePath) diff --git a/src/core/api/reg_gc.go b/src/core/api/reg_gc.go deleted file mode 100644 index 3f13ef5a5..000000000 --- a/src/core/api/reg_gc.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "errors" - "github.com/goharbor/harbor/src/core/config" - "net/http" - "os" - "strconv" - - common_job "github.com/goharbor/harbor/src/common/job" - "github.com/goharbor/harbor/src/core/api/models" -) - -// GCAPI handles request of harbor GC... -type GCAPI struct { - AJAPI -} - -// Prepare validates the URL and parms, it needs the system admin permission. -func (gc *GCAPI) Prepare() { - gc.BaseController.Prepare() - if !gc.SecurityCtx.IsAuthenticated() { - gc.SendUnAuthorizedError(errors.New("UnAuthorized")) - return - } - if !gc.SecurityCtx.IsSysAdmin() { - gc.SendForbiddenError(errors.New(gc.SecurityCtx.GetUsername())) - return - } -} - -// Post according to the request, it creates a cron schedule or a manual trigger for GC. -// create a daily schedule for GC -// { -// "schedule": { -// "type": "Daily", -// "cron": "0 0 0 * * *" -// }, -// "parameters": { -// "delete_untagged": true -// } -// } -// create a manual trigger for GC -// { -// "schedule": { -// "type": "Manual" -// }, -// "parameters": { -// "delete_untagged": true -// "read_only": true -// } -// } -func (gc *GCAPI) Post() { - parameters := make(map[string]interface{}) - ajr := models.AdminJobReq{ - Parameters: parameters, - } - isValid, err := gc.DecodeJSONReqAndValidate(&ajr) - if !isValid { - gc.SendBadRequestError(err) - return - } - ajr.Parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG") - // default is the non-blocking GC job. - ajr.Name = common_job.ImageGC - ajr.Parameters["time_window"] = config.GetGCTimeWindow() - // if specify read_only:true, API will submit the readonly GC job, otherwise default is non-blocking GC job. - readOnlyParam, exist := ajr.Parameters["read_only"] - if exist { - if readOnly, ok := readOnlyParam.(bool); ok && readOnly { - ajr.Name = common_job.ImageGCReadOnly - } - } - gc.submit(&ajr) - gc.Redirect(http.StatusCreated, strconv.FormatInt(ajr.ID, 10)) -} - -// Put handles GC cron schedule update/delete. -// Request: delete the schedule of GC -// { -// "schedule": { -// "type": "None", -// "cron": "" -// }, -// "parameters": { -// "delete_untagged": true -// } -// } -func (gc *GCAPI) Put() { - parameters := make(map[string]interface{}) - ajr := models.AdminJobReq{ - Parameters: parameters, - } - isValid, err := gc.DecodeJSONReqAndValidate(&ajr) - if !isValid { - gc.SendBadRequestError(err) - return - } - ajr.Name = common_job.ImageGC - ajr.Parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG") - ajr.Parameters["time_window"] = config.GetGCTimeWindow() - gc.updateSchedule(ajr) -} - -// GetGC ... -func (gc *GCAPI) GetGC() { - id, err := gc.GetInt64FromPath(":id") - if err != nil { - gc.SendInternalServerError(errors.New("need to specify gc id")) - return - } - gc.get(id) -} - -// List returns the top 10 executions of GC which includes manual and cron. -func (gc *GCAPI) List() { - gc.list(common_job.ImageGC) -} - -// Get gets GC schedule ... -func (gc *GCAPI) Get() { - gc.getSchedule(common_job.ImageGC) -} - -// GetLog ... -func (gc *GCAPI) GetLog() { - id, err := gc.GetInt64FromPath(":id") - if err != nil { - gc.SendBadRequestError(errors.New("invalid ID")) - return - } - gc.getLog(id) -} diff --git a/src/core/api/reg_gc_test.go b/src/core/api/reg_gc_test.go deleted file mode 100644 index 7a55a186d..000000000 --- a/src/core/api/reg_gc_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package api - -import ( - "testing" - - "github.com/goharbor/harbor/src/testing/apitests/apilib" - "github.com/stretchr/testify/assert" -) - -func TestGCPost(t *testing.T) { - - adminJob001 := apilib.AdminJobReq{ - Parameters: map[string]interface{}{"delete_untagged": false}, - } - assert := assert.New(t) - apiTest := newHarborAPI() - - // case 1: add a new admin job - code, err := apiTest.AddGC(*admin, adminJob001) - if err != nil { - t.Error("Error occurred while add a admin job", err.Error()) - t.Log(err) - } else { - assert.Equal(201, code, "Add adminjob status should be 201") - } -} - -func TestGCGet(t *testing.T) { - assert := assert.New(t) - apiTest := newHarborAPI() - - code, _, err := apiTest.GCScheduleGet(*admin) - if err != nil { - t.Error("Error occurred while get a admin job", err.Error()) - t.Log(err) - } else { - assert.Equal(200, code, "Get adminjob status should be 200") - } -} diff --git a/src/pkg/scheduler/scheduler.go b/src/pkg/scheduler/scheduler.go index dc7501abc..de3a06ef0 100644 --- a/src/pkg/scheduler/scheduler.go +++ b/src/pkg/scheduler/scheduler.go @@ -42,6 +42,7 @@ type Schedule struct { VendorID int64 `json:"vendor_id"` CRONType string `json:"cron_type"` CRON string `json:"cron"` + Param string `json:"param"` Status string `json:"status"` // status of the underlying task(jobservice job) CreationTime time.Time `json:"creation_time"` UpdateTime time.Time `json:"update_time"` @@ -269,6 +270,7 @@ func (s *scheduler) convertSchedule(ctx context.Context, schedule *schedule) (*S VendorID: schedule.VendorID, CRONType: schedule.CRONType, CRON: schedule.CRON, + Param: schedule.CallbackFuncParam, CreationTime: schedule.CreationTime, UpdateTime: schedule.UpdateTime, } diff --git a/src/portal/src/lib/components/config/gc/gc-history/gc-history.component.html b/src/portal/src/lib/components/config/gc/gc-history/gc-history.component.html index 0843d93db..bce0cc804 100644 --- a/src/portal/src/lib/components/config/gc/gc-history/gc-history.component.html +++ b/src/portal/src/lib/components/config/gc/gc-history/gc-history.component.html @@ -18,7 +18,7 @@ {{job.createTime | date:'medium'}} {{job.updateTime | date:'medium'}} - + diff --git a/src/server/v2.0/handler/gc.go b/src/server/v2.0/handler/gc.go new file mode 100644 index 000000000..256da13ec --- /dev/null +++ b/src/server/v2.0/handler/gc.go @@ -0,0 +1,120 @@ +package handler + +import ( + "context" + "github.com/go-openapi/runtime/middleware" + "github.com/goharbor/harbor/src/controller/gc" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/pkg/scheduler" + "github.com/goharbor/harbor/src/server/v2.0/handler/model" + "github.com/goharbor/harbor/src/server/v2.0/models" + operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/gc" + "os" +) + +type gcAPI struct { + BaseAPI + gcCtr gc.Controller +} + +func newGCAPI() *gcAPI { + return &gcAPI{ + gcCtr: gc.NewController(), + } +} + +func (g *gcAPI) PostSchedule(ctx context.Context, params operation.PostScheduleParams) middleware.Responder { + if err := g.parseParam(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters); err != nil { + return g.SendError(ctx, err) + } + return operation.NewPostScheduleOK() +} + +func (g *gcAPI) PutSchedule(ctx context.Context, params operation.PutScheduleParams) middleware.Responder { + if err := g.parseParam(ctx, params.Schedule.Schedule.Type, params.Schedule.Schedule.Cron, params.Schedule.Parameters); err != nil { + return g.SendError(ctx, err) + } + return operation.NewPutScheduleOK() +} + +func (g *gcAPI) parseParam(ctx context.Context, scheType string, cron string, parameters map[string]interface{}) error { + // set the required parameters for GC + parameters["redis_url_reg"] = os.Getenv("_REDIS_URL_REG") + parameters["time_window"] = config.GetGCTimeWindow() + + var err error + switch scheType { + case model.ScheduleManual: + err = g.gcCtr.Start(ctx, parameters) + case model.ScheduleNone: + err = g.gcCtr.DeleteSchedule(ctx) + case model.ScheduleHourly, model.ScheduleDaily, model.ScheduleWeekly, model.ScheduleCustom: + err = g.updateSchedule(ctx, cron, parameters) + } + return err +} + +func (g *gcAPI) createSchedule(ctx context.Context, cron string, parameters map[string]interface{}) error { + if cron == "" { + return errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage("empty cron string for gc schedule") + } + _, err := g.gcCtr.CreateSchedule(ctx, cron, parameters) + if err != nil { + return err + } + return nil +} + +func (g *gcAPI) updateSchedule(ctx context.Context, cron string, parameters map[string]interface{}) error { + if err := g.gcCtr.DeleteSchedule(ctx); err != nil { + return err + } + return g.createSchedule(ctx, cron, parameters) +} + +func (g *gcAPI) GetSchedule(ctx context.Context, params operation.GetScheduleParams) middleware.Responder { + schedule, err := g.gcCtr.GetSchedule(ctx) + if errors.IsNotFoundErr(err) { + return operation.NewGetScheduleOK().WithPayload(model.NewSchedule(&scheduler.Schedule{}).ToSwagger()) + } + if err != nil { + return g.SendError(ctx, err) + } + + return operation.NewGetScheduleOK().WithPayload(model.NewSchedule(schedule).ToSwagger()) +} + +func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryParams) middleware.Responder { + query, err := g.BuildQuery(ctx, params.Q, params.Page, params.PageSize) + if err != nil { + return g.SendError(ctx, err) + } + total, err := g.gcCtr.Count(ctx, query) + if err != nil { + return g.SendError(ctx, err) + } + hs, err := g.gcCtr.History(ctx, query) + if err != nil { + return g.SendError(ctx, err) + } + var results []*models.GCHistory + for _, h := range hs { + res := &model.GCHistory{} + res.History = h + results = append(results, res.ToSwagger()) + } + return operation.NewGetGCHistoryOK(). + WithXTotalCount(total). + WithLink(g.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()). + WithPayload(results) +} + +func (g *gcAPI) GetGCLog(ctx context.Context, params operation.GetGCLogParams) middleware.Responder { + log, err := g.gcCtr.GetLog(ctx, params.GcID) + if err != nil { + return g.SendError(ctx, err) + } + return operation.NewGetGCLogOK().WithPayload(string(log)) +} diff --git a/src/server/v2.0/handler/handler.go b/src/server/v2.0/handler/handler.go index 045034446..602aa872c 100644 --- a/src/server/v2.0/handler/handler.go +++ b/src/server/v2.0/handler/handler.go @@ -41,6 +41,7 @@ func New() http.Handler { ReplicationAPI: newReplicationAPI(), SysteminfoAPI: newSystemInfoAPI(), PingAPI: newPingAPI(), + GcAPI: newGCAPI(), }) if err != nil { log.Fatal(err) diff --git a/src/server/v2.0/handler/model/gc.go b/src/server/v2.0/handler/model/gc.go new file mode 100644 index 000000000..3ffa04145 --- /dev/null +++ b/src/server/v2.0/handler/model/gc.go @@ -0,0 +1,75 @@ +package model + +import ( + "github.com/go-openapi/strfmt" + "github.com/goharbor/harbor/src/controller/gc" + "github.com/goharbor/harbor/src/pkg/scheduler" + "github.com/goharbor/harbor/src/server/v2.0/models" +) + +const ( + // ScheduleHourly : 'Hourly' + ScheduleHourly = "Hourly" + // ScheduleDaily : 'Daily' + ScheduleDaily = "Daily" + // ScheduleWeekly : 'Weekly' + ScheduleWeekly = "Weekly" + // ScheduleCustom : 'Custom' + ScheduleCustom = "Custom" + // ScheduleManual : 'Manual' + ScheduleManual = "Manual" + // ScheduleNone : 'None' + ScheduleNone = "None" +) + +// GCHistory ... +type GCHistory struct { + *gc.History +} + +// ToSwagger converts the history to the swagger model +func (h *GCHistory) ToSwagger() *models.GCHistory { + return &models.GCHistory{ + ID: h.ID, + JobName: h.Name, + JobKind: h.Kind, + JobParameters: h.Parameters, + Deleted: h.Deleted, + JobStatus: h.Status, + Schedule: &models.ScheduleObj{ + Cron: h.Schedule.Schedule.Cron, + Type: h.Schedule.Schedule.Type, + }, + CreationTime: strfmt.DateTime(h.CreationTime), + UpdateTime: strfmt.DateTime(h.UpdateTime), + } +} + +// Schedule ... +type Schedule struct { + *scheduler.Schedule +} + +// ToSwagger converts the schedule to the swagger model +// TODO remove the hard code when after issue https://github.com/goharbor/harbor/issues/13047 is resolved. +func (s *Schedule) ToSwagger() *models.GCHistory { + return &models.GCHistory{ + ID: 0, + JobName: "", + JobKind: s.CRON, + JobParameters: s.Param, + Deleted: false, + JobStatus: "", + Schedule: &models.ScheduleObj{ + Cron: s.CRON, + Type: "Custom", + }, + CreationTime: strfmt.DateTime(s.CreationTime), + UpdateTime: strfmt.DateTime(s.UpdateTime), + } +} + +// NewSchedule ... +func NewSchedule(s *scheduler.Schedule) *Schedule { + return &Schedule{Schedule: s} +} diff --git a/src/server/v2.0/route/legacy.go b/src/server/v2.0/route/legacy.go index 1cee7decd..28b675a20 100755 --- a/src/server/v2.0/route/legacy.go +++ b/src/server/v2.0/route/legacy.go @@ -45,10 +45,6 @@ func registerLegacyRoutes() { beego.Router("/api/"+version+"/quotas", &api.QuotaAPI{}, "get:List") beego.Router("/api/"+version+"/quotas/:id([0-9]+)", &api.QuotaAPI{}, "get:Get;put:Put") - beego.Router("/api/"+version+"/system/gc", &api.GCAPI{}, "get:List") - beego.Router("/api/"+version+"/system/gc/:id", &api.GCAPI{}, "get:GetGC") - beego.Router("/api/"+version+"/system/gc/:id([0-9]+)/log", &api.GCAPI{}, "get:GetLog") - beego.Router("/api/"+version+"/system/gc/schedule", &api.GCAPI{}, "get:Get;put:Put;post:Post") beego.Router("/api/"+version+"/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post") beego.Router("/api/"+version+"/system/CVEAllowlist", &api.SysCVEAllowlistAPI{}, "get:Get;put:Put") beego.Router("/api/"+version+"/system/oidc/ping", &api.OIDCAPI{}, "post:Ping")