From ce6ed3eeb7e4075ad8ec32a49eacbb804790a966 Mon Sep 17 00:00:00 2001 From: He Weiwei Date: Mon, 21 Dec 2020 02:17:02 +0000 Subject: [PATCH] refactor(api): move scan all apis to go-swagger Move scan all APIs from beego to go-swagger. Signed-off-by: He Weiwei --- Makefile | 2 +- api/v2.0/legacy_swagger.yaml | 212 ------- api/v2.0/swagger.yaml | 178 +++++- src/core/api/harborapi_test.go | 1 - src/core/api/scan_all.go | 284 ---------- src/core/api/scan_all_test.go | 83 --- src/pkg/scheduler/scheduler.go | 23 +- src/server/v2.0/handler/gc.go | 16 +- src/server/v2.0/handler/handler.go | 4 +- src/server/v2.0/handler/model/gc.go | 31 +- src/server/v2.0/handler/model/schedule.go | 50 ++ src/server/v2.0/handler/scan_all.go | 248 +++++++++ src/server/v2.0/handler/scan_all_test.go | 520 ++++++++++++++++++ src/server/v2.0/route/legacy.go | 6 - src/testing/server/v2.0/handler/handler.go | 158 ++++++ tests/apitests/python/library/base.py | 3 +- tests/apitests/python/library/scan_all.py | 50 ++ tests/apitests/python/library/system.py | 40 -- .../python/test_system_level_scan_all.py | 9 +- 19 files changed, 1249 insertions(+), 669 deletions(-) delete mode 100644 src/core/api/scan_all.go delete mode 100644 src/core/api/scan_all_test.go create mode 100644 src/server/v2.0/handler/model/schedule.go create mode 100644 src/server/v2.0/handler/scan_all.go create mode 100644 src/server/v2.0/handler/scan_all_test.go create mode 100644 src/testing/server/v2.0/handler/handler.go create mode 100644 tests/apitests/python/library/scan_all.py diff --git a/Makefile b/Makefile index f6b92ea3d..909d83cb4 100644 --- a/Makefile +++ b/Makefile @@ -296,7 +296,7 @@ endif SWAGGER_IMAGENAME=goharbor/swagger SWAGGER_VERSION=v0.21.0 SWAGGER=$(DOCKERCMD) run --rm -u $(shell id -u):$(shell id -g) -v $(BUILDPATH):$(BUILDPATH) -w $(BUILDPATH) ${SWAGGER_IMAGENAME}:${SWAGGER_VERSION} -SWAGGER_GENERATE_SERVER=${SWAGGER} generate server --template-dir=$(TOOLSPATH)/swagger/templates --exclude-main --additional-initialism=CVE +SWAGGER_GENERATE_SERVER=${SWAGGER} generate server --template-dir=$(TOOLSPATH)/swagger/templates --exclude-main --additional-initialism=CVE --additional-initialism=GC SWAGGER_IMAGE_BUILD_CMD=${DOCKERBUILD} -f ${TOOLSPATH}/swagger/Dockerfile --build-arg SWAGGER_VERSION=${SWAGGER_VERSION} -t ${SWAGGER_IMAGENAME}:$(SWAGGER_VERSION) . SWAGGER_IMAGENAME: diff --git a/api/v2.0/legacy_swagger.yaml b/api/v2.0/legacy_swagger.yaml index 5e4beefe7..3eec45ad9 100644 --- a/api/v2.0/legacy_swagger.yaml +++ b/api/v2.0/legacy_swagger.yaml @@ -1549,99 +1549,6 @@ paths: description: Only admin has this authority. '500': description: Unexpected internal errors. - '/system/gc/{id}': - get: - summary: Get gc status. - description: This endpoint let user get gc status filtered by specific ID. - parameters: - - name: id - in: path - type: integer - format: int64 - required: true - description: Relevant job ID - tags: - - Products - responses: - '200': - description: Get gc results successfully. - schema: - $ref: '#/definitions/GCResult' - '401': - description: User need to log in first. - '403': - description: User does not have permission of admin role. - '500': - description: Unexpected internal errors. - /system/scanAll/schedule: - get: - summary: Get scan_all's schedule. - description: This endpoint is for getting a schedule for the scan all job, which scans all of images in Harbor. - tags: - - Products - responses: - '200': - description: Get a schedule for the scan all job, which scans all of images in Harbor. - schema: - $ref: '#/definitions/AdminJobSchedule' - '401': - description: User need to log in first. - '403': - description: Only admin has this authority. - '500': - description: Unexpected internal errors. - put: - summary: Update scan all's schedule. - description: | - This endpoint is for updating the schedule of scan all job, which scans all of images in Harbor. - parameters: - - name: schedule - in: body - required: true - schema: - $ref: '#/definitions/AdminJobSchedule' - description: Updates the schedule of scan all job, which scans all of images in Harbor. - tags: - - Products - responses: - '200': - description: Updated scan_all's schedule successfully. - '400': - description: Invalid schedule type. - '401': - description: User need to log in first. - '403': - description: User does not have permission of admin role. - '500': - description: Unexpected internal errors. - post: - summary: Create a schedule or a manual trigger for the scan all job. - description: | - This endpoint is for creating a schedule or a manual trigger for the scan all job, which scans all of images in Harbor. - parameters: - - name: schedule - in: body - required: true - schema: - $ref: '#/definitions/AdminJobSchedule' - description: Create a schedule or a manual trigger for the scan all job. - tags: - - Products - responses: - '200': - description: Updated scan_all's schedule successfully. - '400': - description: Invalid schedule type. - '401': - description: User need to log in first. - '403': - description: User does not have permission of admin role. - '409': - description: There is a "scanall" job in progress, so the request cannot be served. - '500': - description: Unexpected internal errors. - '503': - description: Harbor is not deployed with scanners. /configurations: get: summary: Get system configurations. @@ -3074,42 +2981,6 @@ paths: description: Request is not allowed '500': description: Internal server error happened - '/scans/all/metrics': - get: - summary: Get the metrics of the latest scan all process - description: Get the metrics of the latest scan all process - tags: - - Products - - Scan - responses: - '200': - description: OK - schema: - $ref: '#/definitions/Stats' - '401': - description: Unauthorized request - '403': - description: Request is not allowed - '500': - description: Internal server error happened - '/scans/schedule/metrics': - get: - summary: Get the metrics of the latest scheduled scan all process - description: Get the metrics of the latest scheduled scan all process - tags: - - Products - - Scan - responses: - '200': - description: OK - schema: - $ref: '#/definitions/Stats' - '401': - description: Unauthorized request - '403': - description: Request is not allowed - '500': - description: Internal server error happened responses: OK: description: 'Success' @@ -4152,56 +4023,6 @@ definitions: properties: labels: $ref: '#/definitions/Labels' - GCResult: - type: object - properties: - id: - type: integer - description: the id of gc job. - job_name: - type: string - description: the job name of gc job. - job_kind: - type: string - description: the job kind of gc job. - job_parameters: - type: string - description: the job parameters of gc job. - schedule: - $ref: '#/definitions/AdminJobScheduleObj' - job_status: - type: string - description: the status of gc job. - deleted: - type: boolean - description: if gc job was deleted. - creation_time: - type: string - description: the creation time of gc job. - update_time: - type: string - description: the update time of gc job. - AdminJobSchedule: - type: object - properties: - schedule: - $ref: '#/definitions/AdminJobScheduleObj' - parameters: - type: object - description: The parameters of admin job - additionalProperties: - type: boolean - AdminJobScheduleObj: - type: object - properties: - type: - type: string - description: | - The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manually' and 'None'. - 'Manually' means to trigger it right away and 'None' means to cancel the schedule. - cron: - type: string - description: A cron expression, a time-based job scheduler. SearchResult: type: object description: The chart search result item @@ -4848,39 +4669,6 @@ definitions: type: string description: The identifier of the scanner registration - Stats: - type: object - description: Stats provides the overall progress of the scan all process. - properties: - total: - type: integer - format: int - description: 'The total number of scan processes triggered by the scan all action' - example: 100 - completed: - type: integer - format: int - description: 'The number of the finished scan processes triggered by the scan all action' - example: 90 - requester: - type: string - description: 'The requester identity which usually uses the ID of the scan all job' - example: '28' - metrics: - type: object - description: 'The metrics data for the each status' - additionalProperties: - type: integer - format: int - example: 10 - example: - 'Success': 5 - 'Error': 2, - 'Running': 3 - ongoing: - type: boolean - description: A flag indicating job status of scan all . - SupportedWebhookEventTypes: type: object description: Supportted webhook event types and notify types. diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 06b448841..016ab5a58 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -1976,6 +1976,47 @@ paths: $ref: '#/responses/404' '500': $ref: '#/responses/500' + /scans/all/metrics: + get: + summary: Get the metrics of the latest scan all process + description: Get the metrics of the latest scan all process + tags: + - scanAll + operationId: getLatestScanAllMetrics + responses: + '200': + description: OK + schema: + $ref: '#/definitions/Stats' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '412': + $ref: '#/responses/412' + '500': + $ref: '#/responses/500' + /scans/schedule/metrics: + get: + summary: Get the metrics of the latest scheduled scan all process + description: Get the metrics of the latest scheduled scan all process + tags: + - scanAll + operationId: getLatestScheduledScanAllMetrics + deprecated: true + responses: + '200': + description: OK + schema: + $ref: '#/definitions/Stats' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '412': + $ref: '#/responses/412' + '500': + $ref: '#/responses/500' /systeminfo: get: summary: Get general system info @@ -2181,6 +2222,80 @@ paths: $ref: '#/responses/403' '500': $ref: '#/responses/500' + /system/scanAll/schedule: + get: + summary: Get scan all's schedule. + description: This endpoint is for getting a schedule for the scan all job, which scans all of images in Harbor. + tags: + - scanAll + operationId: getScanAllSchedule + responses: + '200': + description: Get a schedule for the scan all job, which scans all of images in Harbor. + schema: + $ref: '#/definitions/Schedule' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '412': + $ref: '#/responses/412' + '500': + $ref: '#/responses/500' + put: + summary: Update scan all's schedule. + description: This endpoint is for updating the schedule of scan all job, which scans all of images in Harbor. + parameters: + - name: schedule + in: body + required: true + schema: + $ref: '#/definitions/Schedule' + description: Updates the schedule of scan all job, which scans all of images in Harbor. + tags: + - scanAll + operationId: updateScanAllSchedule + responses: + '200': + $ref: '#/responses/200' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '412': + $ref: '#/responses/412' + '500': + $ref: '#/responses/500' + post: + summary: Create a schedule or a manual trigger for the scan all job. + description: This endpoint is for creating a schedule or a manual trigger for the scan all job, which scans all of images in Harbor. + parameters: + - name: schedule + in: body + required: true + schema: + $ref: '#/definitions/Schedule' + description: Create a schedule or a manual trigger for the scan all job. + tags: + - scanAll + operationId: createScanAllSchedule + responses: + '201': + $ref: '#/responses/201' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '409': + $ref: '#/responses/409' + '412': + $ref: '#/responses/412' + '500': + $ref: '#/responses/500' /ping: get: summary: Ping Harbor to check if it's alive. @@ -3563,11 +3678,29 @@ definitions: Schedule: type: object properties: + id: + type: integer + description: The id of the schedule. + readOnly: true + status: + type: string + description: The status of the schedule. + readOnly: true + creation_time: + type: string + format: date-time + description: the creation time of the schedule. + readOnly: true + update_time: + type: string + format: date-time + description: the update time of the schedule. + readOnly: true schedule: $ref: '#/definitions/ScheduleObj' parameters: type: object - description: The parameters of admin job + description: The parameters of schedule job additionalProperties: type: object ScheduleObj: @@ -3576,8 +3709,47 @@ definitions: type: type: string description: | - The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manually' and 'None'. - 'Manually' means to trigger it right away and 'None' means to cancel the schedule. + The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manual' and 'None'. + 'Manual' means to trigger it right away and 'None' means to cancel the schedule. + enum: + - Hourly + - Daily + - Weekly + - Custom + - Manual + - None cron: type: string description: A cron expression, a time-based job scheduler. + + Stats: + type: object + description: Stats provides the overall progress of the scan all process. + properties: + total: + type: integer + format: int + description: 'The total number of scan processes triggered by the scan all action' + example: 100 + x-omitempty: false + completed: + type: integer + format: int + description: 'The number of the finished scan processes triggered by the scan all action' + example: 90 + x-omitempty: false + metrics: + type: object + description: 'The metrics data for the each status' + additionalProperties: + type: integer + format: int + example: 10 + example: + 'Success': 5 + 'Error': 2 + 'Running': 3 + ongoing: + type: boolean + description: A flag indicating job status of scan all. + x-omitempty: false diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 6d6460443..c392a4f6a 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -122,7 +122,6 @@ func init() { beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping") beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List") beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete") - beego.Router("/api/system/scanAll/schedule", &ScanAllAPI{}, "get:Get;put:Put;post:Post") beego.Router("/api/system/CVEAllowlist", &SysCVEAllowlistAPI{}, "get:Get;put:Put") beego.Router("/api/system/oidc/ping", &OIDCAPI{}, "post:Ping") diff --git a/src/core/api/scan_all.go b/src/core/api/scan_all.go deleted file mode 100644 index c9623af22..000000000 --- a/src/core/api/scan_all.go +++ /dev/null @@ -1,284 +0,0 @@ -package api - -import ( - "fmt" - "net/http" - "strconv" - - "github.com/goharbor/harbor/src/controller/scan" - "github.com/goharbor/harbor/src/controller/scanner" - "github.com/goharbor/harbor/src/core/api/models" - "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/lib/errors" - "github.com/goharbor/harbor/src/lib/q" - "github.com/goharbor/harbor/src/pkg/scan/all" - "github.com/goharbor/harbor/src/pkg/scheduler" - "github.com/goharbor/harbor/src/pkg/task" -) - -// ScanAllAPI handles request of scan all images... -type ScanAllAPI struct { - BaseController -} - -// Prepare validates the URL and parms, it needs the system admin permission. -func (sc *ScanAllAPI) Prepare() { - sc.BaseController.Prepare() - - if !sc.SecurityCtx.IsAuthenticated() { - sc.SendUnAuthorizedError(errors.New("UnAuthorized")) - return - } - if !sc.SecurityCtx.IsSysAdmin() { - sc.SendForbiddenError(errors.New(sc.SecurityCtx.GetUsername())) - return - } - - enabled, err := isScanEnabled() - if err != nil { - sc.SendInternalServerError(err) - return - } - - if !enabled { - sc.SendStatusServiceUnavailableError(errors.New("no scanner is configured, it's not possible to scan")) - return - } -} - -// Post according to the request, it creates a cron schedule or a manual trigger for scan all. -// create a daily schedule for scan all -// { -// "schedule": { -// "type": "Daily", -// "cron": "0 0 0 * * *" -// } -// } -// create a manual trigger for scan all -// { -// "schedule": { -// "type": "Manual" -// } -// } -func (sc *ScanAllAPI) Post() { - ajr := models.AdminJobReq{} - isValid, err := sc.DecodeJSONReqAndValidate(&ajr) - if !isValid { - sc.SendBadRequestError(err) - return - } - - if ajr.Schedule == nil { - sc.SendBadRequestError(fmt.Errorf("schedule is required")) - return - } - - if ajr.Schedule.Type == models.ScheduleNone { - return - } - - if ajr.IsPeriodic() { - schedule, err := sc.getScanAllSchedule() - if err != nil { - sc.SendError(err) - return - } - - if schedule != nil { - err := errors.New("fail to set schedule for scan all as always had one, please delete it firstly then to re-schedule") - sc.SendPreconditionFailedError(err) - return - } - - scheduleID, err := sc.createOrUpdateScanAllSchedule(ajr.Schedule.Type, ajr.Schedule.Cron, nil) - if err != nil { - sc.SendError(err) - return - } - - sc.Redirect(http.StatusCreated, strconv.FormatInt(scheduleID, 10)) - } else { - execution, err := sc.getLatestScanAllExecution(task.ExecutionTriggerManual) - if err != nil { - sc.SendError(err) - return - } - - if execution != nil && execution.IsOnGoing() { - err := errors.Errorf("a previous scan all job aleady exits, its status is %s", execution.Status) - sc.SendConflictError(err) - return - } - - executionID, err := scan.DefaultController.ScanAll(sc.Context(), task.ExecutionTriggerManual, true) - if err != nil { - sc.SendError(err) - return - } - - sc.Redirect(http.StatusCreated, strconv.FormatInt(executionID, 10)) - } -} - -// Put handles scan all cron schedule update/delete. -// Request: delete the schedule of scan all -// { -// "schedule": { -// "type": "None", -// "cron": "" -// } -// } -func (sc *ScanAllAPI) Put() { - ajr := models.AdminJobReq{} - isValid, err := sc.DecodeJSONReqAndValidate(&ajr) - if !isValid { - sc.SendBadRequestError(err) - return - } - - if ajr.Schedule.Type == models.ScheduleManual { - err := fmt.Errorf("fail to update scan all schedule as wrong schedule type: %s", ajr.Schedule.Type) - sc.SendBadRequestError(err) - return - } - - schedule, err := sc.getScanAllSchedule() - if err != nil { - sc.SendError(err) - return - } - - if ajr.Schedule.Type == models.ScheduleNone { - if schedule != nil { - err = scheduler.Sched.UnScheduleByID(sc.Context(), schedule.ID) - } - } else { - _, err = sc.createOrUpdateScanAllSchedule(ajr.Schedule.Type, ajr.Schedule.Cron, schedule) - } - - if err != nil { - sc.SendError(err) - } -} - -// Get gets scan all schedule ... -func (sc *ScanAllAPI) Get() { - result := models.AdminJobRep{} - - schedule, err := sc.getScanAllSchedule() - if err != nil { - sc.SendError(err) - return - } - - if schedule != nil { - result.ID = schedule.ID - result.Status = schedule.Status - result.CreationTime = schedule.CreationTime - result.UpdateTime = schedule.UpdateTime - result.Schedule = &models.ScheduleParam{ - Type: schedule.CRONType, - Cron: schedule.CRON, - } - } - - sc.Data["json"] = result - sc.ServeJSON() -} - -// GetScheduleMetrics returns the progress metrics for the latest scheduled scan all job -func (sc *ScanAllAPI) GetScheduleMetrics() { - sc.getMetrics(task.ExecutionTriggerSchedule) -} - -// GetScanAllMetrics returns the progress metrics for the latest manually triggered scan all job -func (sc *ScanAllAPI) GetScanAllMetrics() { - sc.getMetrics(task.ExecutionTriggerManual) -} - -func (sc *ScanAllAPI) getMetrics(trigger string) { - execution, err := sc.getLatestScanAllExecution(trigger) - if err != nil { - sc.SendError(err) - return - } - - sts := &all.Stats{} - if execution != nil && execution.Metrics != nil { - metrics := execution.Metrics - sts.Total = uint(metrics.TaskCount) - sts.Completed = uint(metrics.SuccessTaskCount) - sts.Metrics = map[string]uint{ - "Pending": uint(metrics.PendingTaskCount), - "Running": uint(metrics.RunningTaskCount), - "Success": uint(metrics.SuccessTaskCount), - "Error": uint(metrics.ErrorTaskCount), - "Stopped": uint(metrics.StoppedTaskCount), - } - sts.Ongoing = !job.Status(execution.Status).Final() || sts.Total != sts.Completed - } - - sc.Data["json"] = sts - sc.ServeJSON() -} - -func (sc *ScanAllAPI) getScanAllSchedule() (*scheduler.Schedule, error) { - query := q.New(q.KeyWords{"vendor_type": job.ImageScanAllJob}) - schedules, err := scheduler.Sched.ListSchedules(sc.Context(), query.First("-creation_time")) - if err != nil { - return nil, err - } - - if len(schedules) > 1 { - msg := "found more than one scheduled scan all job, please ensure that only one schedule left" - return nil, errors.BadRequestError(nil).WithMessage(msg) - } else if len(schedules) == 0 { - return nil, nil - } - - return schedules[0], nil -} - -func (sc *ScanAllAPI) createOrUpdateScanAllSchedule(cronType, cron string, previous *scheduler.Schedule) (int64, error) { - if previous != nil { - if cronType == previous.CRONType && cron == previous.CRON { - return previous.ID, nil - } - - if err := scheduler.Sched.UnScheduleByID(sc.Context(), previous.ID); err != nil { - return 0, err - } - } - - return scheduler.Sched.Schedule(sc.Context(), job.ImageScanAllJob, 0, cronType, cron, scan.ScanAllCallback, nil, nil) -} - -func (sc *ScanAllAPI) getLatestScanAllExecution(trigger string) (*task.Execution, error) { - query := q.New(q.KeyWords{"vendor_type": job.ImageScanAllJob, "trigger": trigger}) - executions, err := task.ExecMgr.List(sc.Context(), query.First("-start_time")) - if err != nil { - return nil, err - } - - if len(executions) == 0 { - return nil, nil - } - - return executions[0], nil -} - -func isScanEnabled() (bool, error) { - kws := make(map[string]interface{}) - kws["is_default"] = true - - query := &q.Query{ - Keywords: kws, - } - - l, err := scanner.DefaultController.ListRegistrations(query) - if err != nil { - return false, errors.Wrap(err, "scan all API: check if scan is enabled") - } - - return len(l) > 0, nil -} diff --git a/src/core/api/scan_all_test.go b/src/core/api/scan_all_test.go deleted file mode 100644 index 2aa69a483..000000000 --- a/src/core/api/scan_all_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "testing" - - "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" - sc "github.com/goharbor/harbor/src/pkg/scan/scanner" - "github.com/goharbor/harbor/src/testing/apitests/apilib" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" -) - -var adminJob002 apilib.AdminJobReq - -// ScanAllAPITestSuite is a test suite to test scan all API. -type ScanAllAPITestSuite struct { - suite.Suite - - m sc.Manager - uuid string -} - -// TestScanAllAPI is an entry point for ScanAllAPITestSuite. -func TestScanAllAPI(t *testing.T) { - suite.Run(t, &ScanAllAPITestSuite{}) -} - -// SetupSuite prepares env for test suite. -func (suite *ScanAllAPITestSuite) SetupSuite() { - // Ensure scanner is there - reg := &scanner.Registration{ - Name: "Trivy", - Description: "The trivy scanner adapter", - URL: "https://trivy.com:8080", - Disabled: false, - IsDefault: true, - } - - scMgr := sc.New() - uuid, err := scMgr.Create(reg) - require.NoError(suite.T(), err, "failed to initialize trivy scanner") - - suite.uuid = uuid - suite.m = scMgr -} - -// TearDownSuite clears env for the test suite. -func (suite *ScanAllAPITestSuite) TearDownSuite() { - err := suite.m.Delete(suite.uuid) - suite.NoError(err, "clear scanner") -} - -func (suite *ScanAllAPITestSuite) TestScanAllPost() { - apiTest := newHarborAPI() - - // case 1: add a new scan all job - adminJob002.Schedule = &apilib.ScheduleParam{Type: "Manual"} - code, err := apiTest.AddScanAll(*admin, adminJob002) - require.NoError(suite.T(), err, "Error occurred while add a scan all job") - suite.Equal(201, code, "Add scan all status should be 200") -} - -func (suite *ScanAllAPITestSuite) TestScanAllGet() { - apiTest := newHarborAPI() - - code, _, err := apiTest.ScanAllScheduleGet(*admin) - require.NoError(suite.T(), err, "Error occurred while get a scan all job") - suite.Equal(200, code, "Get scan all status should be 200") -} diff --git a/src/pkg/scheduler/scheduler.go b/src/pkg/scheduler/scheduler.go index c79252fc5..d83ebf9ea 100644 --- a/src/pkg/scheduler/scheduler.go +++ b/src/pkg/scheduler/scheduler.go @@ -136,20 +136,19 @@ func (s *scheduler) schedule(ctx context.Context, vendorType string, vendorID in CreationTime: now, UpdateTime: now, } - if params != nil { - paramsData, err := json.Marshal(params) - if err != nil { - return 0, err - } - sched.CallbackFuncParam = string(paramsData) + + paramsData, err := json.Marshal(params) + if err != nil { + return 0, err } - if extras != nil { - extrasData, err := json.Marshal(extras) - if err != nil { - return 0, err - } - sched.ExtraAttrs = string(extrasData) + sched.CallbackFuncParam = string(paramsData) + + extrasData, err := json.Marshal(extras) + if err != nil { + return 0, err } + sched.ExtraAttrs = string(extrasData) + // create schedule record // when checkin hook comes, the database record must exist, // so the database record must be created first before submitting job diff --git a/src/server/v2.0/handler/gc.go b/src/server/v2.0/handler/gc.go index 22e8ab8e3..b468db06e 100644 --- a/src/server/v2.0/handler/gc.go +++ b/src/server/v2.0/handler/gc.go @@ -4,18 +4,18 @@ import ( "context" "encoding/json" "fmt" + "os" + "strings" + "github.com/go-openapi/runtime/middleware" "github.com/goharbor/harbor/src/controller/gc" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/q" - "github.com/goharbor/harbor/src/pkg/scheduler" "github.com/goharbor/harbor/src/pkg/task" "github.com/goharbor/harbor/src/server/v2.0/handler/model" "github.com/goharbor/harbor/src/server/v2.0/models" operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/gc" - "os" - "strings" ) type gcAPI struct { @@ -116,13 +116,13 @@ func (g *gcAPI) updateSchedule(ctx context.Context, cronType, cron string, polic func (g *gcAPI) GetGCSchedule(ctx context.Context, params operation.GetGCScheduleParams) middleware.Responder { schedule, err := g.gcCtr.GetSchedule(ctx) if errors.IsNotFoundErr(err) { - return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(&scheduler.Schedule{}).ToSwagger()) + return operation.NewGetGCScheduleOK() } if err != nil { return g.SendError(ctx, err) } - return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(schedule).ToSwagger()) + return operation.NewGetGCScheduleOK().WithPayload(model.NewGCSchedule(schedule).ToSwagger()) } func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryParams) middleware.Responder { @@ -171,7 +171,7 @@ func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryP } func (g *gcAPI) GetGC(ctx context.Context, params operation.GetGCParams) middleware.Responder { - exec, err := g.gcCtr.GetExecution(ctx, params.GcID) + exec, err := g.gcCtr.GetExecution(ctx, params.GCID) if err != nil { return g.SendError(ctx, err) } @@ -199,13 +199,13 @@ func (g *gcAPI) GetGC(ctx context.Context, params operation.GetGCParams) middlew func (g *gcAPI) GetGCLog(ctx context.Context, params operation.GetGCLogParams) middleware.Responder { tasks, err := g.gcCtr.ListTasks(ctx, q.New(q.KeyWords{ - "ExecutionID": params.GcID, + "ExecutionID": params.GCID, })) if err != nil { return g.SendError(ctx, err) } if len(tasks) == 0 { - return g.SendError(ctx, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("garbage collection %d log is not found", params.GcID)) + return g.SendError(ctx, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("garbage collection %d log is not found", params.GCID)) } log, err := g.gcCtr.GetTaskLog(ctx, tasks[0].ID) if err != nil { diff --git a/src/server/v2.0/handler/handler.go b/src/server/v2.0/handler/handler.go index 602aa872c..850006f57 100644 --- a/src/server/v2.0/handler/handler.go +++ b/src/server/v2.0/handler/handler.go @@ -19,7 +19,6 @@ import ( "net/http" lib_http "github.com/goharbor/harbor/src/lib/http" - "github.com/goharbor/harbor/src/server/middleware" "github.com/goharbor/harbor/src/server/middleware/blob" "github.com/goharbor/harbor/src/server/middleware/quota" @@ -33,6 +32,7 @@ func New() http.Handler { RepositoryAPI: newRepositoryAPI(), AuditlogAPI: newAuditLogAPI(), ScanAPI: newScanAPI(), + ScanAllAPI: newScanAllAPI(), ProjectAPI: newProjectAPI(), PreheatAPI: newPreheatAPI(), IconAPI: newIconAPI(), @@ -41,7 +41,7 @@ func New() http.Handler { ReplicationAPI: newReplicationAPI(), SysteminfoAPI: newSystemInfoAPI(), PingAPI: newPingAPI(), - GcAPI: newGCAPI(), + GCAPI: newGCAPI(), }) if err != nil { log.Fatal(err) diff --git a/src/server/v2.0/handler/model/gc.go b/src/server/v2.0/handler/model/gc.go index 6b2052cb7..b6665a0df 100644 --- a/src/server/v2.0/handler/model/gc.go +++ b/src/server/v2.0/handler/model/gc.go @@ -2,11 +2,13 @@ package model import ( "encoding/json" + "strings" + "time" + "github.com/go-openapi/strfmt" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/scheduler" "github.com/goharbor/harbor/src/server/v2.0/models" - "time" ) // ScheduleParam defines the parameter of schedule trigger @@ -41,44 +43,49 @@ func (h *GCHistory) ToSwagger() *models.GCHistory { Deleted: h.Deleted, JobStatus: h.Status, Schedule: &models.ScheduleObj{ + // covert MANUAL to Manual because the type of the ScheduleObj + // must be 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manual' and 'None' + Type: strings.Title(strings.ToLower(h.Schedule.Type)), Cron: h.Schedule.Cron, - Type: h.Schedule.Type, }, CreationTime: strfmt.DateTime(h.CreationTime), UpdateTime: strfmt.DateTime(h.UpdateTime), } } -// Schedule ... -type Schedule struct { +// GCSchedule ... +type GCSchedule struct { *scheduler.Schedule } // ToSwagger converts the schedule to the swagger model -// TODO remove the hard code when after issue https://github.com/goharbor/harbor/issues/13047 is resolved. -func (s *Schedule) ToSwagger() *models.GCHistory { +func (s *GCSchedule) ToSwagger() *models.GCHistory { + if s.Schedule == nil { + return nil + } + e, err := json.Marshal(s.ExtraAttrs) if err != nil { log.Error(err) } return &models.GCHistory{ - ID: 0, + ID: s.ID, JobName: "", JobKind: s.CRON, JobParameters: string(e), Deleted: false, - JobStatus: "", + JobStatus: s.Status, Schedule: &models.ScheduleObj{ Cron: s.CRON, - Type: "Custom", + Type: s.CRONType, }, CreationTime: strfmt.DateTime(s.CreationTime), UpdateTime: strfmt.DateTime(s.UpdateTime), } } -// NewSchedule ... -func NewSchedule(s *scheduler.Schedule) *Schedule { - return &Schedule{Schedule: s} +// NewGCSchedule ... +func NewGCSchedule(s *scheduler.Schedule) *GCSchedule { + return &GCSchedule{Schedule: s} } diff --git a/src/server/v2.0/handler/model/schedule.go b/src/server/v2.0/handler/model/schedule.go new file mode 100644 index 000000000..6f60078aa --- /dev/null +++ b/src/server/v2.0/handler/model/schedule.go @@ -0,0 +1,50 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "github.com/go-openapi/strfmt" + "github.com/goharbor/harbor/src/pkg/scheduler" + "github.com/goharbor/harbor/src/server/v2.0/models" +) + +// Schedule model +type Schedule struct { + *scheduler.Schedule +} + +// ToSwagger converts the schedule to the swagger model +func (s *Schedule) ToSwagger() *models.Schedule { + if s.Schedule == nil { + return nil + } + + return &models.Schedule{ + ID: s.ID, + Status: s.Status, + Schedule: &models.ScheduleObj{ + Cron: s.CRON, + Type: s.CRONType, + }, + Parameters: s.ExtraAttrs, + CreationTime: strfmt.DateTime(s.CreationTime), + UpdateTime: strfmt.DateTime(s.UpdateTime), + } +} + +// NewSchedule new schedule instance +func NewSchedule(schedule *scheduler.Schedule) *Schedule { + return &Schedule{Schedule: schedule} +} diff --git a/src/server/v2.0/handler/scan_all.go b/src/server/v2.0/handler/scan_all.go new file mode 100644 index 000000000..c5dff5db2 --- /dev/null +++ b/src/server/v2.0/handler/scan_all.go @@ -0,0 +1,248 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handler + +import ( + "context" + "fmt" + + "github.com/go-openapi/runtime/middleware" + "github.com/goharbor/harbor/src/controller/scan" + "github.com/goharbor/harbor/src/controller/scanner" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/scheduler" + "github.com/goharbor/harbor/src/pkg/task" + "github.com/goharbor/harbor/src/server/v2.0/handler/model" + "github.com/goharbor/harbor/src/server/v2.0/models" + operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/scan_all" +) + +func newScanAllAPI() *scanAllAPI { + return &scanAllAPI{ + execMgr: task.ExecMgr, + scanCtl: scan.DefaultController, + scannerCtl: scanner.DefaultController, + scheduler: scheduler.Sched, + } +} + +type scanAllAPI struct { + BaseAPI + execMgr task.ExecutionManager + scanCtl scan.Controller + scannerCtl scanner.Controller + scheduler scheduler.Scheduler +} + +func (s *scanAllAPI) Prepare(ctx context.Context, operation string, params interface{}) middleware.Responder { + if err := s.RequireSysAdmin(ctx); err != nil { + return s.SendError(ctx, err) + } + + if err := s.requireScanEnabled(ctx); err != nil { + return s.SendError(ctx, err) + } + + return nil +} + +func (s *scanAllAPI) CreateScanAllSchedule(ctx context.Context, params operation.CreateScanAllScheduleParams) middleware.Responder { + req := params.Schedule + + if req.Schedule.Type == ScheduleNone { + return operation.NewCreateScanAllScheduleCreated() + } + + if req.Schedule.Type == ScheduleManual { + execution, err := s.getLatestScanAllExecution(ctx, task.ExecutionTriggerManual) + if err != nil { + return s.SendError(ctx, err) + } + + if execution != nil && execution.IsOnGoing() { + message := fmt.Sprintf("a previous scan all job aleady exits, its status is %s", execution.Status) + return s.SendError(ctx, errors.ConflictError(nil).WithMessage(message)) + } + + if _, err := s.scanCtl.ScanAll(ctx, task.ExecutionTriggerManual, true); err != nil { + return s.SendError(ctx, err) + } + } else { + schedule, err := s.getScanAllSchedule(ctx) + if err != nil { + return s.SendError(ctx, err) + } + + if schedule != nil { + message := "fail to set schedule for scan all as always had one, please delete it firstly then to re-schedule" + return s.SendError(ctx, errors.PreconditionFailedError(nil).WithMessage(message)) + } + + if _, err := s.createOrUpdateScanAllSchedule(ctx, req.Schedule.Type, req.Schedule.Cron, nil); err != nil { + return s.SendError(ctx, err) + } + } + + return operation.NewCreateScanAllScheduleCreated() +} + +func (s *scanAllAPI) UpdateScanAllSchedule(ctx context.Context, params operation.UpdateScanAllScheduleParams) middleware.Responder { + req := params.Schedule + + if req.Schedule.Type == ScheduleManual { + message := fmt.Sprintf("fail to update scan all schedule as wrong schedule type: %s", req.Schedule.Type) + return s.SendError(ctx, errors.BadRequestError(nil).WithMessage(message)) + } + + schedule, err := s.getScanAllSchedule(ctx) + if err != nil { + return s.SendError(ctx, err) + } + + if req.Schedule.Type == ScheduleNone { + if schedule != nil { + err = s.scheduler.UnScheduleByID(ctx, schedule.ID) + } + } else { + _, err = s.createOrUpdateScanAllSchedule(ctx, req.Schedule.Type, req.Schedule.Cron, schedule) + } + + if err != nil { + return s.SendError(ctx, err) + } + + return operation.NewUpdateScanAllScheduleOK() +} + +func (s *scanAllAPI) GetScanAllSchedule(ctx context.Context, params operation.GetScanAllScheduleParams) middleware.Responder { + schedule, err := s.getScanAllSchedule(ctx) + if err != nil { + return s.SendError(ctx, err) + } + + return operation.NewGetScanAllScheduleOK().WithPayload(model.NewSchedule(schedule).ToSwagger()) +} + +func (s *scanAllAPI) GetLatestScanAllMetrics(ctx context.Context, params operation.GetLatestScanAllMetricsParams) middleware.Responder { + stats, err := s.getMetrics(ctx) + if err != nil { + return s.SendError(ctx, err) + } + + return operation.NewGetLatestScanAllMetricsOK().WithPayload(stats) +} + +func (s *scanAllAPI) GetLatestScheduledScanAllMetrics(ctx context.Context, params operation.GetLatestScheduledScanAllMetricsParams) middleware.Responder { + stats, err := s.getMetrics(ctx, task.ExecutionTriggerSchedule) + if err != nil { + return s.SendError(ctx, err) + } + + return operation.NewGetLatestScanAllMetricsOK().WithPayload(stats) +} + +func (s *scanAllAPI) createOrUpdateScanAllSchedule(ctx context.Context, cronType, cron string, previous *scheduler.Schedule) (int64, error) { + if previous != nil { + if cronType == previous.CRONType && cron == previous.CRON { + return previous.ID, nil + } + + if err := s.scheduler.UnScheduleByID(ctx, previous.ID); err != nil { + return 0, err + } + } + + return s.scheduler.Schedule(ctx, job.ImageScanAllJob, 0, cronType, cron, scan.ScanAllCallback, nil, nil) +} + +func (s *scanAllAPI) getScanAllSchedule(ctx context.Context) (*scheduler.Schedule, error) { + query := q.New(q.KeyWords{"vendor_type": job.ImageScanAllJob}) + schedules, err := s.scheduler.ListSchedules(ctx, query.First("-creation_time")) + if err != nil { + return nil, err + } + + if len(schedules) > 1 { + return nil, fmt.Errorf("found more than one scheduled scan all job, please ensure that only one schedule left") + } else if len(schedules) == 0 { + return nil, nil + } + + return schedules[0], nil +} + +func (s *scanAllAPI) getMetrics(ctx context.Context, trigger ...string) (*models.Stats, error) { + execution, err := s.getLatestScanAllExecution(ctx, trigger...) + if err != nil { + return nil, err + } + + sts := &models.Stats{} + if execution != nil && execution.Metrics != nil { + metrics := execution.Metrics + sts.Total = metrics.TaskCount + sts.Completed = metrics.SuccessTaskCount + sts.Metrics = map[string]int64{ + "Pending": metrics.PendingTaskCount, + "Running": metrics.RunningTaskCount, + "Success": metrics.SuccessTaskCount, + "Error": metrics.ErrorTaskCount, + "Stopped": metrics.StoppedTaskCount, + } + sts.Ongoing = !job.Status(execution.Status).Final() || sts.Total != sts.Completed + } + + return sts, nil +} + +func (s *scanAllAPI) getLatestScanAllExecution(ctx context.Context, trigger ...string) (*task.Execution, error) { + query := q.New(q.KeyWords{"vendor_type": job.ImageScanAllJob}) + if len(trigger) > 0 { + query.Keywords["trigger"] = trigger[0] + } + + executions, err := s.execMgr.List(ctx, query.First("-start_time")) + if err != nil { + return nil, err + } + + if len(executions) == 0 { + return nil, nil + } + + return executions[0], nil +} + +func (s *scanAllAPI) requireScanEnabled(ctx context.Context) error { + kws := make(map[string]interface{}) + kws["is_default"] = true + + query := &q.Query{ + Keywords: kws, + } + + l, err := s.scannerCtl.ListRegistrations(query) + if err != nil { + return errors.Wrap(err, "check if scan is enabled") + } + + if len(l) == 0 { + return errors.PreconditionFailedError(nil).WithMessage("no scanner is configured, it's not possible to scan") + } + + return nil +} diff --git a/src/server/v2.0/handler/scan_all_test.go b/src/server/v2.0/handler/scan_all_test.go new file mode 100644 index 000000000..cb3bbca8f --- /dev/null +++ b/src/server/v2.0/handler/scan_all_test.go @@ -0,0 +1,520 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handler + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "testing" + "time" + + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scheduler" + "github.com/goharbor/harbor/src/pkg/task" + taskdao "github.com/goharbor/harbor/src/pkg/task/dao" + "github.com/goharbor/harbor/src/server/v2.0/models" + "github.com/goharbor/harbor/src/server/v2.0/restapi" + scantesting "github.com/goharbor/harbor/src/testing/controller/scan" + scannertesting "github.com/goharbor/harbor/src/testing/controller/scanner" + "github.com/goharbor/harbor/src/testing/mock" + schedulertesting "github.com/goharbor/harbor/src/testing/pkg/scheduler" + tasktesting "github.com/goharbor/harbor/src/testing/pkg/task" + htesting "github.com/goharbor/harbor/src/testing/server/v2.0/handler" + "github.com/stretchr/testify/suite" +) + +type ScanAllTestSuite struct { + htesting.Suite + + execMgr *tasktesting.ExecutionManager + scanCtl *scantesting.Controller + scannerCtl *scannertesting.Controller + scheduler *schedulertesting.Scheduler + + execution *task.Execution + schedule *scheduler.Schedule +} + +func (suite *ScanAllTestSuite) SetupSuite() { + suite.execution = &task.Execution{ + Status: "Running", + Metrics: &taskdao.Metrics{ + TaskCount: 10, + SuccessTaskCount: 5, + ErrorTaskCount: 0, + PendingTaskCount: 4, + RunningTaskCount: 1, + ScheduledTaskCount: 0, + StoppedTaskCount: 0, + }, + } + + suite.schedule = &scheduler.Schedule{ + ID: 1, + VendorType: "vendor_type", + CRONType: "Daily", + CRON: "0 0 0 * * *", + Status: "Running", + CreationTime: time.Now(), + UpdateTime: time.Now(), + } + + suite.execMgr = &tasktesting.ExecutionManager{} + suite.scanCtl = &scantesting.Controller{} + suite.scannerCtl = &scannertesting.Controller{} + suite.scheduler = &schedulertesting.Scheduler{} + + suite.Config = &restapi.Config{ + ScanAllAPI: &scanAllAPI{ + execMgr: suite.execMgr, + scanCtl: suite.scanCtl, + scannerCtl: suite.scannerCtl, + scheduler: suite.scheduler, + }, + } + + suite.Suite.SetupSuite() +} + +func (suite *ScanAllTestSuite) TestAuthorization() { + newBody := func(body interface{}) io.Reader { + if body == nil { + return nil + } + + buf, err := json.Marshal(body) + suite.Require().NoError(err) + return bytes.NewBuffer(buf) + } + + schedule := models.Schedule{ + Schedule: &models.ScheduleObj{Type: "Manual"}, + } + + reqs := []struct { + method string + url string + body interface{} + }{ + {http.MethodGet, "/scans/all/metrics", nil}, + {http.MethodGet, "/scans/schedule/metrics", nil}, + {http.MethodGet, "/system/scanAll/schedule", nil}, + {http.MethodPut, "/system/scanAll/schedule", schedule}, + {http.MethodPost, "/system/scanAll/schedule", schedule}, + } + for _, req := range reqs { + { + // authorized required + suite.Security.On("IsAuthenticated").Return(false).Once() + + res, err := suite.DoReq(req.method, req.url, newBody(req.body)) + suite.NoError(err) + suite.Equal(401, res.StatusCode) + } + + { + // system admin required + suite.Security.On("IsAuthenticated").Return(true).Once() + suite.Security.On("IsSysAdmin").Return(false).Once() + suite.Security.On("GetUsername").Return("username").Once() + + res, err := suite.DoReq(req.method, req.url, newBody(req.body)) + suite.NoError(err) + suite.Equal(403, res.StatusCode) + } + + { + // default scanner required + suite.Security.On("IsAuthenticated").Return(true).Once() + suite.Security.On("IsSysAdmin").Return(true).Once() + + mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return(nil, nil).Once() + + res, err := suite.DoReq(req.method, req.url, newBody(req.body)) + suite.NoError(err) + suite.Equal(412, res.StatusCode) + } + + { + // default scanner required failed + suite.Security.On("IsAuthenticated").Return(true).Once() + suite.Security.On("IsSysAdmin").Return(true).Once() + + mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return(nil, fmt.Errorf("failed")).Once() + + res, err := suite.DoReq(req.method, req.url, newBody(req.body)) + suite.NoError(err) + suite.Equal(500, res.StatusCode) + } + } +} + +func (suite *ScanAllTestSuite) TestGetLatestScanAllMetrics() { + times := 3 + suite.Security.On("IsAuthenticated").Return(true).Times(times) + suite.Security.On("IsSysAdmin").Return(true).Times(times) + mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times) + + { + // get scan all execution failed + mock.OnAnything(suite.execMgr, "List").Return(nil, fmt.Errorf("failed to list executions")).Once() + + res, err := suite.Get("/scans/all/metrics") + suite.NoError(err) + suite.Equal(500, res.StatusCode) + } + + { + // scan all execution not found + mock.OnAnything(suite.execMgr, "List").Return(nil, nil).Once() + + var stats map[string]interface{} + res, err := suite.GetJSON("/scans/all/metrics", &stats) + suite.NoError(err) + suite.Equal(200, res.StatusCode) + suite.Contains(stats, "ongoing") + } + + { + // scan all execution found + mock.OnAnything(suite.execMgr, "List").Return([]*task.Execution{suite.execution}, nil).Once() + + var stats models.Stats + res, err := suite.GetJSON("/scans/all/metrics", &stats) + suite.NoError(err) + suite.Equal(200, res.StatusCode) + suite.True(stats.Ongoing) + } +} + +func (suite *ScanAllTestSuite) TestGetLatestScheduledScanAllMetrics() { + times := 3 + suite.Security.On("IsAuthenticated").Return(true).Times(times) + suite.Security.On("IsSysAdmin").Return(true).Times(times) + mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times) + + { + // get scan all execution failed + mock.OnAnything(suite.execMgr, "List").Return(nil, fmt.Errorf("failed to list executions")).Once() + + res, err := suite.Get("/scans/schedule/metrics") + suite.NoError(err) + suite.Equal(500, res.StatusCode) + } + + { + // scan all execution not found + mock.OnAnything(suite.execMgr, "List").Return(nil, nil).Once() + + var stats map[string]interface{} + res, err := suite.GetJSON("/scans/schedule/metrics", &stats) + suite.NoError(err) + suite.Equal(200, res.StatusCode) + suite.Contains(stats, "ongoing") + } + + { + // scan all execution found + mock.OnAnything(suite.execMgr, "List").Return([]*task.Execution{suite.execution}, nil).Once() + + var stats models.Stats + res, err := suite.GetJSON("/scans/schedule/metrics", &stats) + suite.NoError(err) + suite.Equal(200, res.StatusCode) + suite.True(stats.Ongoing) + } +} + +func (suite *ScanAllTestSuite) TestCreateScanAllSchedule() { + times := 11 + suite.Security.On("IsAuthenticated").Return(true).Times(times) + suite.Security.On("IsSysAdmin").Return(true).Times(times) + mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times) + + { + // create scan all schedule no body + res, err := suite.Post("/system/scanAll/schedule", nil) + suite.NoError(err) + suite.Equal(422, res.StatusCode) + } + + { + // create scan all schedule with bad body + res, err := suite.Post("/system/scanAll/schedule", bytes.NewBuffer([]byte("bad body"))) + suite.NoError(err) + suite.Equal(422, res.StatusCode) + } + + { + // create scan all schedule with ScheduleNone + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}} + res, err := suite.PostJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(201, res.StatusCode) + } + + { + // create scan all schedule with ScheduleManual but get latest scan all execution failed + mock.OnAnything(suite.execMgr, "List").Return(nil, fmt.Errorf("list executions failed")).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}} + res, err := suite.PostJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(500, res.StatusCode) + } + + { + // create scan all schedule with ScheduleManual but a previous scan all job aleady exits + mock.OnAnything(suite.execMgr, "List").Return([]*task.Execution{suite.execution}, nil).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}} + res, err := suite.PostJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(409, res.StatusCode) + } + + { + // create scan all schedule with ScheduleManual no previous scan all job exits + mock.OnAnything(suite.execMgr, "List").Return(nil, nil).Once() + mock.OnAnything(suite.scanCtl, "ScanAll").Return(int64(1), nil).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}} + res, err := suite.PostJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(201, res.StatusCode) + } + + { + // create scan all schedule with ScheduleManual but scan all failed + mock.OnAnything(suite.execMgr, "List").Return(nil, nil).Once() + mock.OnAnything(suite.scanCtl, "ScanAll").Return(int64(0), fmt.Errorf("scan all failed")).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}} + res, err := suite.PostJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(500, res.StatusCode) + } + + { + // create scan all schedule with periodic but get latest schedule failed + mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, fmt.Errorf("get schedule failed")).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}} + res, err := suite.PostJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(500, res.StatusCode) + } + + { + // create scan all schedule with periodic but schedule areadly exists + mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}} + res, err := suite.PostJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(412, res.StatusCode) + } + + { + // create scan all schedule with periodic but create schedule failed + mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, nil).Once() + mock.OnAnything(suite.scheduler, "Schedule").Return(int64(0), fmt.Errorf("create schedule failed")).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}} + res, err := suite.PostJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(500, res.StatusCode) + } + + { + // create scan all schedule with periodic + mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, nil).Once() + mock.OnAnything(suite.scheduler, "Schedule").Return(int64(1), nil).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}} + res, err := suite.PostJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(201, res.StatusCode) + } +} + +func (suite *ScanAllTestSuite) TestUpdateScanAllSchedule() { + times := 11 + suite.Security.On("IsAuthenticated").Return(true).Times(times) + suite.Security.On("IsSysAdmin").Return(true).Times(times) + mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times) + + { + // update scan all schedule no body + res, err := suite.Put("/system/scanAll/schedule", nil) + suite.NoError(err) + suite.Equal(422, res.StatusCode) + } + + { + // update scan all schedule with bad body + res, err := suite.Put("/system/scanAll/schedule", bytes.NewBuffer([]byte("bad body"))) + suite.NoError(err) + suite.Equal(422, res.StatusCode) + } + + { + // update scan all schedule with ScheduleManual + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}} + res, err := suite.PutJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(400, res.StatusCode) + } + + { + // update scan all schedule but get schedule failed + mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, fmt.Errorf("get schedule failed")).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}} + res, err := suite.PutJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(500, res.StatusCode) + } + + { + // update scan all schedule with ScheduleNone when no schedule found + mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, nil).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}} + res, err := suite.PutJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(200, res.StatusCode) + } + + { + // update scan all schedule with ScheduleNone and unschedule failed + mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once() + mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(fmt.Errorf("unschedule failed")).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}} + res, err := suite.PutJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(500, res.StatusCode) + } + + { + // update scan all schedule with ScheduleNone successfully + mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once() + mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(nil).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}} + res, err := suite.PutJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(200, res.StatusCode) + } + + { + // update scan all schedule with periodic but schedule not changed + mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}} + res, err := suite.PutJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(200, res.StatusCode) + } + + { + // update scan all schedule with periodic and schedule changed + mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once() + mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(nil).Once() + mock.OnAnything(suite.scheduler, "Schedule").Return(int64(1), nil).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleCustom, Cron: "0 1 0 * * *"}} + res, err := suite.PutJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(200, res.StatusCode) + } + + { + // update scan all schedule with periodic and schedule changed, but unschedule old schedule failed + mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once() + mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(fmt.Errorf("unschedule failed")).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleCustom, Cron: "0 1 0 * * *"}} + res, err := suite.PutJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(500, res.StatusCode) + } + + { + // update scan all schedule with periodic and schedule changed, but creat new schedule failed + mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once() + mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(nil).Once() + mock.OnAnything(suite.scheduler, "Schedule").Return(int64(0), fmt.Errorf("create schedule failed")).Once() + + body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleCustom, Cron: "0 1 0 * * *"}} + res, err := suite.PutJSON("/system/scanAll/schedule", body) + suite.NoError(err) + suite.Equal(500, res.StatusCode) + } +} + +func (suite *ScanAllTestSuite) TestGetScanAllSchedule() { + times := 4 + suite.Security.On("IsAuthenticated").Return(true).Times(times) + suite.Security.On("IsSysAdmin").Return(true).Times(times) + mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times) + + { + // get schedule failed + mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, fmt.Errorf("get schedule failed")).Once() + + res, err := suite.Get("/system/scanAll/schedule") + suite.NoError(err) + suite.Equal(500, res.StatusCode) + } + + { + // schedule not found + mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, nil).Once() + + res, err := suite.Get("/system/scanAll/schedule") + suite.NoError(err) + suite.Equal(200, res.StatusCode) + } + + { + // schedule found + mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once() + + var schedule models.Schedule + res, err := suite.GetJSON("/system/scanAll/schedule", &schedule) + suite.NoError(err) + suite.Equal(200, res.StatusCode) + suite.Equal(suite.schedule.CRONType, schedule.Schedule.Type) + suite.Equal(suite.schedule.CRON, schedule.Schedule.Cron) + } + + { + // schedule found more than one + mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule, suite.schedule}, nil).Once() + + res, err := suite.Get("/system/scanAll/schedule") + suite.NoError(err) + suite.Equal(500, res.StatusCode) + } +} + +func TestScanAllTestSuite(t *testing.T) { + suite.Run(t, &ScanAllTestSuite{}) +} diff --git a/src/server/v2.0/route/legacy.go b/src/server/v2.0/route/legacy.go index 28b675a20..f6168063c 100755 --- a/src/server/v2.0/route/legacy.go +++ b/src/server/v2.0/route/legacy.go @@ -45,7 +45,6 @@ func registerLegacyRoutes() { beego.Router("/api/"+version+"/quotas", &api.QuotaAPI{}, "get:List") beego.Router("/api/"+version+"/quotas/:id([0-9]+)", &api.QuotaAPI{}, "get:Get;put:Put") - beego.Router("/api/"+version+"/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post") beego.Router("/api/"+version+"/system/CVEAllowlist", &api.SysCVEAllowlistAPI{}, "get:Get;put:Put") beego.Router("/api/"+version+"/system/oidc/ping", &api.OIDCAPI{}, "post:Ping") @@ -106,9 +105,4 @@ func registerLegacyRoutes() { proScannerAPI := &api.ProjectScannerAPI{} beego.Router("/api/"+version+"/projects/:pid([0-9]+)/scanner", proScannerAPI, "get:GetProjectScanner;put:SetProjectScanner") beego.Router("/api/"+version+"/projects/:pid([0-9]+)/scanner/candidates", proScannerAPI, "get:GetProScannerCandidates") - - // Add routes for scan all metrics - scanAllAPI := &api.ScanAllAPI{} - beego.Router("/api/"+version+"/scans/all/metrics", scanAllAPI, "get:GetScanAllMetrics") - beego.Router("/api/"+version+"/scans/schedule/metrics", scanAllAPI, "get:GetScheduleMetrics") } diff --git a/src/testing/server/v2.0/handler/handler.go b/src/testing/server/v2.0/handler/handler.go new file mode 100644 index 000000000..4de64079b --- /dev/null +++ b/src/testing/server/v2.0/handler/handler.go @@ -0,0 +1,158 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handler + +import ( + "bytes" + "encoding/json" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + + "github.com/goharbor/harbor/src/common/security" + lib "github.com/goharbor/harbor/src/lib/http" + "github.com/goharbor/harbor/src/server/middleware" + "github.com/goharbor/harbor/src/server/v2.0/restapi" + securitytesting "github.com/goharbor/harbor/src/testing/common/security" + "github.com/stretchr/testify/suite" +) + +// Suite ... +type Suite struct { + suite.Suite + + Config *restapi.Config + Security *securitytesting.Context + ts *httptest.Server + tc *http.Client +} + +// SetupSuite ... +func (suite *Suite) SetupSuite() { + h, api, _ := restapi.HandlerAPI(*suite.Config) + + api.ServeError = func(rw http.ResponseWriter, r *http.Request, err error) { + lib.SendError(rw, err) + } + + suite.Security = &securitytesting.Context{} + m := middleware.New(func(w http.ResponseWriter, r *http.Request, next http.Handler) { + next.ServeHTTP(w, r.WithContext(security.NewContext(r.Context(), suite.Security))) + }) + + suite.ts = httptest.NewServer(m(h)) + suite.tc = http.DefaultClient +} + +// TearDownSuite ... +func (suite *Suite) TearDownSuite() { + suite.ts.Close() +} + +// DoReq ... +func (suite *Suite) DoReq(method string, url string, body io.Reader, contentTypes ...string) (*http.Response, error) { + req, err := http.NewRequest(method, suite.ts.URL+"/api/v2.0"+url, body) + if err != nil { + return nil, err + } + + contentType := "application/json" + if len(contentTypes) > 0 { + contentType = contentTypes[0] + } + req.Header.Set("Content-Type", contentType) + + return suite.tc.Do(req) +} + +// Delete ... +func (suite *Suite) Delete(url string, contentTypes ...string) (*http.Response, error) { + return suite.DoReq(http.MethodDelete, url, nil, contentTypes...) +} + +// Get ... +func (suite *Suite) Get(url string, contentTypes ...string) (*http.Response, error) { + return suite.DoReq(http.MethodGet, url, nil, contentTypes...) +} + +// GetJSON ... +func (suite *Suite) GetJSON(url string, js interface{}) (*http.Response, error) { + res, err := suite.Get(url) + if err != nil { + return nil, err + } + + if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest { + data, err := ioutil.ReadAll(res.Body) + if err != nil { + return res, err + } + res.Body.Close() + + if err := json.Unmarshal(data, js); err != nil { + return res, err + } + + res.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + } + + return res, nil +} + +// Patch ... +func (suite *Suite) Patch(url string, body io.Reader, contentTypes ...string) (*http.Response, error) { + return suite.DoReq(http.MethodPatch, url, body, contentTypes...) +} + +// PatchJSON ... +func (suite *Suite) PatchJSON(url string, js interface{}) (*http.Response, error) { + buf, err := json.Marshal(js) + if err != nil { + return nil, err + } + + return suite.Patch(url, bytes.NewBuffer(buf)) +} + +// Post ... +func (suite *Suite) Post(url string, body io.Reader, contentTypes ...string) (*http.Response, error) { + return suite.DoReq(http.MethodPost, url, body, contentTypes...) +} + +// PostJSON ... +func (suite *Suite) PostJSON(url string, js interface{}) (*http.Response, error) { + buf, err := json.Marshal(js) + if err != nil { + return nil, err + } + + return suite.Post(url, bytes.NewBuffer(buf)) +} + +// Put ... +func (suite *Suite) Put(url string, body io.Reader, contentTypes ...string) (*http.Response, error) { + return suite.DoReq(http.MethodPut, url, body, contentTypes...) +} + +// PutJSON ... +func (suite *Suite) PutJSON(url string, js interface{}) (*http.Response, error) { + buf, err := json.Marshal(js) + if err != nil { + return nil, err + } + + return suite.Put(url, bytes.NewBuffer(buf)) +} diff --git a/tests/apitests/python/library/base.py b/tests/apitests/python/library/base.py index 1f5cfa16d..5bb956f5c 100644 --- a/tests/apitests/python/library/base.py +++ b/tests/apitests/python/library/base.py @@ -28,7 +28,7 @@ def get_endpoint(): def _create_client(server, credential, debug, api_type="products"): cfg = None - if api_type in ('projectv2', 'artifact', 'repository', 'scan', 'preheat', 'replication', 'robot', 'gc'): + if api_type in ('projectv2', 'artifact', 'repository', 'scan', 'scanall', 'preheat', 'replication', 'robot', 'gc'): cfg = v2_swagger_client.Configuration() else: cfg = swagger_client.Configuration() @@ -58,6 +58,7 @@ def _create_client(server, credential, debug, api_type="products"): "preheat": v2_swagger_client.PreheatApi(v2_swagger_client.ApiClient(cfg)), "repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)), "scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)), + "scanall": v2_swagger_client.ScanAllApi(v2_swagger_client.ApiClient(cfg)), "scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)), "replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)), "robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)), diff --git a/tests/apitests/python/library/scan_all.py b/tests/apitests/python/library/scan_all.py new file mode 100644 index 000000000..ca1a48595 --- /dev/null +++ b/tests/apitests/python/library/scan_all.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +import time +import base +import v2_swagger_client +from v2_swagger_client.rest import ApiException + +class ScanAll(base.Base): + def __init__(self): + super(ScanAll,self).__init__(api_type="scanall") + + def create_scan_all_schedule(self, schedule_type, cron=None, expect_status_code=201, expect_response_body=None, **kwargs): + client = self._get_client(**kwargs) + + schedule_obj = v2_swagger_client.ScheduleObj() + schedule_obj.type = schedule_type + if cron is not None: + schedule_obj.cron = cron + + schedule = v2_swagger_client.Schedule() + schedule.schedule = schedule_obj + + try: + _, status_code, _ = client.create_scan_all_schedule_with_http_info(schedule) + except ApiException as e: + if e.status == expect_status_code: + if expect_response_body is not None and e.body.strip() != expect_response_body.strip(): + raise Exception(r"Create scan all schedule response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip())) + else: + return e.reason, e.body + else: + raise Exception(r"Create scan all schedule result is not as expected {} actual status is {}.".format(expect_status_code, e.status)) + base._assert_status_code(expect_status_code, status_code) + + def scan_all_now(self, **kwargs): + self.create_scan_all_schedule('Manual', **kwargs) + + def wait_until_scans_all_finish(self, **kwargs): + client = self._get_client(**kwargs) + timeout_count = 50 + while True: + time.sleep(5) + timeout_count = timeout_count - 1 + if (timeout_count == 0): + break + stats = client.get_latest_scan_all_metrics() + print("Scan all status:", stats) + if stats.ongoing is False: + return + raise Exception("Error: Scan all job is timeout.") diff --git a/tests/apitests/python/library/system.py b/tests/apitests/python/library/system.py index 19ae28a93..fd60303ac 100644 --- a/tests/apitests/python/library/system.py +++ b/tests/apitests/python/library/system.py @@ -117,46 +117,6 @@ class System(base.Base): base._assert_status_code(expect_status_code, status_code) return base._get_id_from_header(header) - def wait_until_scans_all_finish(self, **kwargs): - client = self._get_client(**kwargs) - timeout_count = 50 - scan_status="" - while True: - time.sleep(5) - timeout_count = timeout_count - 1 - if (timeout_count == 0): - break - stats = client.scans_all_metrics_get() - print("Scan all status:", stats) - if stats.ongoing is False: - return - raise Exception("Error: Scan all job is timeout.") - - def create_scan_all_schedule(self, schedule_type, cron = None, expect_status_code = 201, expect_response_body = None, **kwargs): - client = self._get_client(**kwargs) - scanschedule = swagger_client.AdminJobScheduleObj() - scanschedule.type = schedule_type - if cron is not None: - scanschedule.cron = cron - - scan_all_schedule = swagger_client.AdminJobSchedule(scanschedule) - try: - _, status_code, header = client.system_scan_all_schedule_post_with_http_info(scan_all_schedule) - except ApiException as e: - if e.status == expect_status_code: - if expect_response_body is not None and e.body.strip() != expect_response_body.strip(): - raise Exception(r"Create Scan All schedule response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip())) - else: - return e.reason, e.body - else: - raise Exception(r"Create Scan All schedule result is not as expected {} actual status is {}.".format(expect_status_code, e.status)) - base._assert_status_code(expect_status_code, status_code) - return base._get_id_from_header(header) - - def scan_now(self, **kwargs): - scan_all_id = self.create_scan_all_schedule('Manual', **kwargs) - return scan_all_id - def set_cve_allowlist(self, expires_at=None, expected_status_code=200, *cve_ids, **kwargs): client = self._get_client(**kwargs) cve_list = [swagger_client.CVEAllowlistItem(cve_id=c) for c in cve_ids] diff --git a/tests/apitests/python/test_system_level_scan_all.py b/tests/apitests/python/test_system_level_scan_all.py index dc3822d23..60b78c71e 100644 --- a/tests/apitests/python/test_system_level_scan_all.py +++ b/tests/apitests/python/test_system_level_scan_all.py @@ -4,20 +4,21 @@ import unittest from testutils import harbor_server, suppress_urllib3_warning from testutils import TEARDOWN from testutils import ADMIN_CLIENT -from library.system import System from library.project import Project from library.user import User from library.repository import Repository from library.repository import push_self_build_image_to_project from library.artifact import Artifact +from library.scan_all import ScanAll + class TestScanAll(unittest.TestCase): @suppress_urllib3_warning def setUp(self): - self.system = System() self.project= Project() self.user= User() self.artifact = Artifact() self.repo = Repository() + self.scan_all = ScanAll() @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") def tearDown(self): @@ -81,8 +82,8 @@ class TestScanAll(unittest.TestCase): TestScanAll.repo_Luca_name, tag_Luca = push_self_build_image_to_project(TestScanAll.project_Luca_name, harbor_server, user_Luca_name, user_common_password, image_b, src_tag) #4. Trigger scan all event; - self.system.scan_now(**ADMIN_CLIENT) - self.system.wait_until_scans_all_finish(**ADMIN_CLIENT) + self.scan_all.scan_all_now(**ADMIN_CLIENT) + self.scan_all.wait_until_scans_all_finish(**ADMIN_CLIENT) #5. Check if image in project_Alice and another image in project_Luca were both scanned. self.artifact.check_image_scan_result(TestScanAll.project_Alice_name, image_a, tag_Alice, **USER_ALICE_CLIENT)