refactor(api): move scan all apis to go-swagger

Move scan all APIs from beego to go-swagger.

Signed-off-by: He Weiwei <hweiwei@vmware.com>
This commit is contained in:
He Weiwei 2020-12-21 02:17:02 +00:00
parent 9bc6f3cee4
commit ce6ed3eeb7
19 changed files with 1249 additions and 669 deletions

View File

@ -296,7 +296,7 @@ endif
SWAGGER_IMAGENAME=goharbor/swagger SWAGGER_IMAGENAME=goharbor/swagger
SWAGGER_VERSION=v0.21.0 SWAGGER_VERSION=v0.21.0
SWAGGER=$(DOCKERCMD) run --rm -u $(shell id -u):$(shell id -g) -v $(BUILDPATH):$(BUILDPATH) -w $(BUILDPATH) ${SWAGGER_IMAGENAME}:${SWAGGER_VERSION} SWAGGER=$(DOCKERCMD) run --rm -u $(shell id -u):$(shell id -g) -v $(BUILDPATH):$(BUILDPATH) -w $(BUILDPATH) ${SWAGGER_IMAGENAME}:${SWAGGER_VERSION}
SWAGGER_GENERATE_SERVER=${SWAGGER} generate server --template-dir=$(TOOLSPATH)/swagger/templates --exclude-main --additional-initialism=CVE SWAGGER_GENERATE_SERVER=${SWAGGER} generate server --template-dir=$(TOOLSPATH)/swagger/templates --exclude-main --additional-initialism=CVE --additional-initialism=GC
SWAGGER_IMAGE_BUILD_CMD=${DOCKERBUILD} -f ${TOOLSPATH}/swagger/Dockerfile --build-arg SWAGGER_VERSION=${SWAGGER_VERSION} -t ${SWAGGER_IMAGENAME}:$(SWAGGER_VERSION) . SWAGGER_IMAGE_BUILD_CMD=${DOCKERBUILD} -f ${TOOLSPATH}/swagger/Dockerfile --build-arg SWAGGER_VERSION=${SWAGGER_VERSION} -t ${SWAGGER_IMAGENAME}:$(SWAGGER_VERSION) .
SWAGGER_IMAGENAME: SWAGGER_IMAGENAME:

View File

@ -1549,99 +1549,6 @@ paths:
description: Only admin has this authority. description: Only admin has this authority.
'500': '500':
description: Unexpected internal errors. description: Unexpected internal errors.
'/system/gc/{id}':
get:
summary: Get gc status.
description: This endpoint let user get gc status filtered by specific ID.
parameters:
- name: id
in: path
type: integer
format: int64
required: true
description: Relevant job ID
tags:
- Products
responses:
'200':
description: Get gc results successfully.
schema:
$ref: '#/definitions/GCResult'
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'500':
description: Unexpected internal errors.
/system/scanAll/schedule:
get:
summary: Get scan_all's schedule.
description: This endpoint is for getting a schedule for the scan all job, which scans all of images in Harbor.
tags:
- Products
responses:
'200':
description: Get a schedule for the scan all job, which scans all of images in Harbor.
schema:
$ref: '#/definitions/AdminJobSchedule'
'401':
description: User need to log in first.
'403':
description: Only admin has this authority.
'500':
description: Unexpected internal errors.
put:
summary: Update scan all's schedule.
description: |
This endpoint is for updating the schedule of scan all job, which scans all of images in Harbor.
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/AdminJobSchedule'
description: Updates the schedule of scan all job, which scans all of images in Harbor.
tags:
- Products
responses:
'200':
description: Updated scan_all's schedule successfully.
'400':
description: Invalid schedule type.
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'500':
description: Unexpected internal errors.
post:
summary: Create a schedule or a manual trigger for the scan all job.
description: |
This endpoint is for creating a schedule or a manual trigger for the scan all job, which scans all of images in Harbor.
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/AdminJobSchedule'
description: Create a schedule or a manual trigger for the scan all job.
tags:
- Products
responses:
'200':
description: Updated scan_all's schedule successfully.
'400':
description: Invalid schedule type.
'401':
description: User need to log in first.
'403':
description: User does not have permission of admin role.
'409':
description: There is a "scanall" job in progress, so the request cannot be served.
'500':
description: Unexpected internal errors.
'503':
description: Harbor is not deployed with scanners.
/configurations: /configurations:
get: get:
summary: Get system configurations. summary: Get system configurations.
@ -3074,42 +2981,6 @@ paths:
description: Request is not allowed description: Request is not allowed
'500': '500':
description: Internal server error happened description: Internal server error happened
'/scans/all/metrics':
get:
summary: Get the metrics of the latest scan all process
description: Get the metrics of the latest scan all process
tags:
- Products
- Scan
responses:
'200':
description: OK
schema:
$ref: '#/definitions/Stats'
'401':
description: Unauthorized request
'403':
description: Request is not allowed
'500':
description: Internal server error happened
'/scans/schedule/metrics':
get:
summary: Get the metrics of the latest scheduled scan all process
description: Get the metrics of the latest scheduled scan all process
tags:
- Products
- Scan
responses:
'200':
description: OK
schema:
$ref: '#/definitions/Stats'
'401':
description: Unauthorized request
'403':
description: Request is not allowed
'500':
description: Internal server error happened
responses: responses:
OK: OK:
description: 'Success' description: 'Success'
@ -4152,56 +4023,6 @@ definitions:
properties: properties:
labels: labels:
$ref: '#/definitions/Labels' $ref: '#/definitions/Labels'
GCResult:
type: object
properties:
id:
type: integer
description: the id of gc job.
job_name:
type: string
description: the job name of gc job.
job_kind:
type: string
description: the job kind of gc job.
job_parameters:
type: string
description: the job parameters of gc job.
schedule:
$ref: '#/definitions/AdminJobScheduleObj'
job_status:
type: string
description: the status of gc job.
deleted:
type: boolean
description: if gc job was deleted.
creation_time:
type: string
description: the creation time of gc job.
update_time:
type: string
description: the update time of gc job.
AdminJobSchedule:
type: object
properties:
schedule:
$ref: '#/definitions/AdminJobScheduleObj'
parameters:
type: object
description: The parameters of admin job
additionalProperties:
type: boolean
AdminJobScheduleObj:
type: object
properties:
type:
type: string
description: |
The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manually' and 'None'.
'Manually' means to trigger it right away and 'None' means to cancel the schedule.
cron:
type: string
description: A cron expression, a time-based job scheduler.
SearchResult: SearchResult:
type: object type: object
description: The chart search result item description: The chart search result item
@ -4848,39 +4669,6 @@ definitions:
type: string type: string
description: The identifier of the scanner registration description: The identifier of the scanner registration
Stats:
type: object
description: Stats provides the overall progress of the scan all process.
properties:
total:
type: integer
format: int
description: 'The total number of scan processes triggered by the scan all action'
example: 100
completed:
type: integer
format: int
description: 'The number of the finished scan processes triggered by the scan all action'
example: 90
requester:
type: string
description: 'The requester identity which usually uses the ID of the scan all job'
example: '28'
metrics:
type: object
description: 'The metrics data for the each status'
additionalProperties:
type: integer
format: int
example: 10
example:
'Success': 5
'Error': 2,
'Running': 3
ongoing:
type: boolean
description: A flag indicating job status of scan all .
SupportedWebhookEventTypes: SupportedWebhookEventTypes:
type: object type: object
description: Supportted webhook event types and notify types. description: Supportted webhook event types and notify types.

View File

@ -1976,6 +1976,47 @@ paths:
$ref: '#/responses/404' $ref: '#/responses/404'
'500': '500':
$ref: '#/responses/500' $ref: '#/responses/500'
/scans/all/metrics:
get:
summary: Get the metrics of the latest scan all process
description: Get the metrics of the latest scan all process
tags:
- scanAll
operationId: getLatestScanAllMetrics
responses:
'200':
description: OK
schema:
$ref: '#/definitions/Stats'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'412':
$ref: '#/responses/412'
'500':
$ref: '#/responses/500'
/scans/schedule/metrics:
get:
summary: Get the metrics of the latest scheduled scan all process
description: Get the metrics of the latest scheduled scan all process
tags:
- scanAll
operationId: getLatestScheduledScanAllMetrics
deprecated: true
responses:
'200':
description: OK
schema:
$ref: '#/definitions/Stats'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'412':
$ref: '#/responses/412'
'500':
$ref: '#/responses/500'
/systeminfo: /systeminfo:
get: get:
summary: Get general system info summary: Get general system info
@ -2181,6 +2222,80 @@ paths:
$ref: '#/responses/403' $ref: '#/responses/403'
'500': '500':
$ref: '#/responses/500' $ref: '#/responses/500'
/system/scanAll/schedule:
get:
summary: Get scan all's schedule.
description: This endpoint is for getting a schedule for the scan all job, which scans all of images in Harbor.
tags:
- scanAll
operationId: getScanAllSchedule
responses:
'200':
description: Get a schedule for the scan all job, which scans all of images in Harbor.
schema:
$ref: '#/definitions/Schedule'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'412':
$ref: '#/responses/412'
'500':
$ref: '#/responses/500'
put:
summary: Update scan all's schedule.
description: This endpoint is for updating the schedule of scan all job, which scans all of images in Harbor.
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/Schedule'
description: Updates the schedule of scan all job, which scans all of images in Harbor.
tags:
- scanAll
operationId: updateScanAllSchedule
responses:
'200':
$ref: '#/responses/200'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'412':
$ref: '#/responses/412'
'500':
$ref: '#/responses/500'
post:
summary: Create a schedule or a manual trigger for the scan all job.
description: This endpoint is for creating a schedule or a manual trigger for the scan all job, which scans all of images in Harbor.
parameters:
- name: schedule
in: body
required: true
schema:
$ref: '#/definitions/Schedule'
description: Create a schedule or a manual trigger for the scan all job.
tags:
- scanAll
operationId: createScanAllSchedule
responses:
'201':
$ref: '#/responses/201'
'400':
$ref: '#/responses/400'
'401':
$ref: '#/responses/401'
'403':
$ref: '#/responses/403'
'409':
$ref: '#/responses/409'
'412':
$ref: '#/responses/412'
'500':
$ref: '#/responses/500'
/ping: /ping:
get: get:
summary: Ping Harbor to check if it's alive. summary: Ping Harbor to check if it's alive.
@ -3563,11 +3678,29 @@ definitions:
Schedule: Schedule:
type: object type: object
properties: properties:
id:
type: integer
description: The id of the schedule.
readOnly: true
status:
type: string
description: The status of the schedule.
readOnly: true
creation_time:
type: string
format: date-time
description: the creation time of the schedule.
readOnly: true
update_time:
type: string
format: date-time
description: the update time of the schedule.
readOnly: true
schedule: schedule:
$ref: '#/definitions/ScheduleObj' $ref: '#/definitions/ScheduleObj'
parameters: parameters:
type: object type: object
description: The parameters of admin job description: The parameters of schedule job
additionalProperties: additionalProperties:
type: object type: object
ScheduleObj: ScheduleObj:
@ -3576,8 +3709,47 @@ definitions:
type: type:
type: string type: string
description: | description: |
The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manually' and 'None'. The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manual' and 'None'.
'Manually' means to trigger it right away and 'None' means to cancel the schedule. 'Manual' means to trigger it right away and 'None' means to cancel the schedule.
enum:
- Hourly
- Daily
- Weekly
- Custom
- Manual
- None
cron: cron:
type: string type: string
description: A cron expression, a time-based job scheduler. description: A cron expression, a time-based job scheduler.
Stats:
type: object
description: Stats provides the overall progress of the scan all process.
properties:
total:
type: integer
format: int
description: 'The total number of scan processes triggered by the scan all action'
example: 100
x-omitempty: false
completed:
type: integer
format: int
description: 'The number of the finished scan processes triggered by the scan all action'
example: 90
x-omitempty: false
metrics:
type: object
description: 'The metrics data for the each status'
additionalProperties:
type: integer
format: int
example: 10
example:
'Success': 5
'Error': 2
'Running': 3
ongoing:
type: boolean
description: A flag indicating job status of scan all.
x-omitempty: false

View File

@ -122,7 +122,6 @@ func init() {
beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping") beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping")
beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List") beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List")
beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete") beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete")
beego.Router("/api/system/scanAll/schedule", &ScanAllAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/system/CVEAllowlist", &SysCVEAllowlistAPI{}, "get:Get;put:Put") beego.Router("/api/system/CVEAllowlist", &SysCVEAllowlistAPI{}, "get:Get;put:Put")
beego.Router("/api/system/oidc/ping", &OIDCAPI{}, "post:Ping") beego.Router("/api/system/oidc/ping", &OIDCAPI{}, "post:Ping")

View File

@ -1,284 +0,0 @@
package api
import (
"fmt"
"net/http"
"strconv"
"github.com/goharbor/harbor/src/controller/scan"
"github.com/goharbor/harbor/src/controller/scanner"
"github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/scan/all"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
)
// ScanAllAPI handles request of scan all images...
type ScanAllAPI struct {
BaseController
}
// Prepare validates the URL and parms, it needs the system admin permission.
func (sc *ScanAllAPI) Prepare() {
sc.BaseController.Prepare()
if !sc.SecurityCtx.IsAuthenticated() {
sc.SendUnAuthorizedError(errors.New("UnAuthorized"))
return
}
if !sc.SecurityCtx.IsSysAdmin() {
sc.SendForbiddenError(errors.New(sc.SecurityCtx.GetUsername()))
return
}
enabled, err := isScanEnabled()
if err != nil {
sc.SendInternalServerError(err)
return
}
if !enabled {
sc.SendStatusServiceUnavailableError(errors.New("no scanner is configured, it's not possible to scan"))
return
}
}
// Post according to the request, it creates a cron schedule or a manual trigger for scan all.
// create a daily schedule for scan all
// {
// "schedule": {
// "type": "Daily",
// "cron": "0 0 0 * * *"
// }
// }
// create a manual trigger for scan all
// {
// "schedule": {
// "type": "Manual"
// }
// }
func (sc *ScanAllAPI) Post() {
ajr := models.AdminJobReq{}
isValid, err := sc.DecodeJSONReqAndValidate(&ajr)
if !isValid {
sc.SendBadRequestError(err)
return
}
if ajr.Schedule == nil {
sc.SendBadRequestError(fmt.Errorf("schedule is required"))
return
}
if ajr.Schedule.Type == models.ScheduleNone {
return
}
if ajr.IsPeriodic() {
schedule, err := sc.getScanAllSchedule()
if err != nil {
sc.SendError(err)
return
}
if schedule != nil {
err := errors.New("fail to set schedule for scan all as always had one, please delete it firstly then to re-schedule")
sc.SendPreconditionFailedError(err)
return
}
scheduleID, err := sc.createOrUpdateScanAllSchedule(ajr.Schedule.Type, ajr.Schedule.Cron, nil)
if err != nil {
sc.SendError(err)
return
}
sc.Redirect(http.StatusCreated, strconv.FormatInt(scheduleID, 10))
} else {
execution, err := sc.getLatestScanAllExecution(task.ExecutionTriggerManual)
if err != nil {
sc.SendError(err)
return
}
if execution != nil && execution.IsOnGoing() {
err := errors.Errorf("a previous scan all job aleady exits, its status is %s", execution.Status)
sc.SendConflictError(err)
return
}
executionID, err := scan.DefaultController.ScanAll(sc.Context(), task.ExecutionTriggerManual, true)
if err != nil {
sc.SendError(err)
return
}
sc.Redirect(http.StatusCreated, strconv.FormatInt(executionID, 10))
}
}
// Put handles scan all cron schedule update/delete.
// Request: delete the schedule of scan all
// {
// "schedule": {
// "type": "None",
// "cron": ""
// }
// }
func (sc *ScanAllAPI) Put() {
ajr := models.AdminJobReq{}
isValid, err := sc.DecodeJSONReqAndValidate(&ajr)
if !isValid {
sc.SendBadRequestError(err)
return
}
if ajr.Schedule.Type == models.ScheduleManual {
err := fmt.Errorf("fail to update scan all schedule as wrong schedule type: %s", ajr.Schedule.Type)
sc.SendBadRequestError(err)
return
}
schedule, err := sc.getScanAllSchedule()
if err != nil {
sc.SendError(err)
return
}
if ajr.Schedule.Type == models.ScheduleNone {
if schedule != nil {
err = scheduler.Sched.UnScheduleByID(sc.Context(), schedule.ID)
}
} else {
_, err = sc.createOrUpdateScanAllSchedule(ajr.Schedule.Type, ajr.Schedule.Cron, schedule)
}
if err != nil {
sc.SendError(err)
}
}
// Get gets scan all schedule ...
func (sc *ScanAllAPI) Get() {
result := models.AdminJobRep{}
schedule, err := sc.getScanAllSchedule()
if err != nil {
sc.SendError(err)
return
}
if schedule != nil {
result.ID = schedule.ID
result.Status = schedule.Status
result.CreationTime = schedule.CreationTime
result.UpdateTime = schedule.UpdateTime
result.Schedule = &models.ScheduleParam{
Type: schedule.CRONType,
Cron: schedule.CRON,
}
}
sc.Data["json"] = result
sc.ServeJSON()
}
// GetScheduleMetrics returns the progress metrics for the latest scheduled scan all job
func (sc *ScanAllAPI) GetScheduleMetrics() {
sc.getMetrics(task.ExecutionTriggerSchedule)
}
// GetScanAllMetrics returns the progress metrics for the latest manually triggered scan all job
func (sc *ScanAllAPI) GetScanAllMetrics() {
sc.getMetrics(task.ExecutionTriggerManual)
}
func (sc *ScanAllAPI) getMetrics(trigger string) {
execution, err := sc.getLatestScanAllExecution(trigger)
if err != nil {
sc.SendError(err)
return
}
sts := &all.Stats{}
if execution != nil && execution.Metrics != nil {
metrics := execution.Metrics
sts.Total = uint(metrics.TaskCount)
sts.Completed = uint(metrics.SuccessTaskCount)
sts.Metrics = map[string]uint{
"Pending": uint(metrics.PendingTaskCount),
"Running": uint(metrics.RunningTaskCount),
"Success": uint(metrics.SuccessTaskCount),
"Error": uint(metrics.ErrorTaskCount),
"Stopped": uint(metrics.StoppedTaskCount),
}
sts.Ongoing = !job.Status(execution.Status).Final() || sts.Total != sts.Completed
}
sc.Data["json"] = sts
sc.ServeJSON()
}
func (sc *ScanAllAPI) getScanAllSchedule() (*scheduler.Schedule, error) {
query := q.New(q.KeyWords{"vendor_type": job.ImageScanAllJob})
schedules, err := scheduler.Sched.ListSchedules(sc.Context(), query.First("-creation_time"))
if err != nil {
return nil, err
}
if len(schedules) > 1 {
msg := "found more than one scheduled scan all job, please ensure that only one schedule left"
return nil, errors.BadRequestError(nil).WithMessage(msg)
} else if len(schedules) == 0 {
return nil, nil
}
return schedules[0], nil
}
func (sc *ScanAllAPI) createOrUpdateScanAllSchedule(cronType, cron string, previous *scheduler.Schedule) (int64, error) {
if previous != nil {
if cronType == previous.CRONType && cron == previous.CRON {
return previous.ID, nil
}
if err := scheduler.Sched.UnScheduleByID(sc.Context(), previous.ID); err != nil {
return 0, err
}
}
return scheduler.Sched.Schedule(sc.Context(), job.ImageScanAllJob, 0, cronType, cron, scan.ScanAllCallback, nil, nil)
}
func (sc *ScanAllAPI) getLatestScanAllExecution(trigger string) (*task.Execution, error) {
query := q.New(q.KeyWords{"vendor_type": job.ImageScanAllJob, "trigger": trigger})
executions, err := task.ExecMgr.List(sc.Context(), query.First("-start_time"))
if err != nil {
return nil, err
}
if len(executions) == 0 {
return nil, nil
}
return executions[0], nil
}
func isScanEnabled() (bool, error) {
kws := make(map[string]interface{})
kws["is_default"] = true
query := &q.Query{
Keywords: kws,
}
l, err := scanner.DefaultController.ListRegistrations(query)
if err != nil {
return false, errors.Wrap(err, "scan all API: check if scan is enabled")
}
return len(l) > 0, nil
}

View File

@ -1,83 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"testing"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
sc "github.com/goharbor/harbor/src/pkg/scan/scanner"
"github.com/goharbor/harbor/src/testing/apitests/apilib"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
var adminJob002 apilib.AdminJobReq
// ScanAllAPITestSuite is a test suite to test scan all API.
type ScanAllAPITestSuite struct {
suite.Suite
m sc.Manager
uuid string
}
// TestScanAllAPI is an entry point for ScanAllAPITestSuite.
func TestScanAllAPI(t *testing.T) {
suite.Run(t, &ScanAllAPITestSuite{})
}
// SetupSuite prepares env for test suite.
func (suite *ScanAllAPITestSuite) SetupSuite() {
// Ensure scanner is there
reg := &scanner.Registration{
Name: "Trivy",
Description: "The trivy scanner adapter",
URL: "https://trivy.com:8080",
Disabled: false,
IsDefault: true,
}
scMgr := sc.New()
uuid, err := scMgr.Create(reg)
require.NoError(suite.T(), err, "failed to initialize trivy scanner")
suite.uuid = uuid
suite.m = scMgr
}
// TearDownSuite clears env for the test suite.
func (suite *ScanAllAPITestSuite) TearDownSuite() {
err := suite.m.Delete(suite.uuid)
suite.NoError(err, "clear scanner")
}
func (suite *ScanAllAPITestSuite) TestScanAllPost() {
apiTest := newHarborAPI()
// case 1: add a new scan all job
adminJob002.Schedule = &apilib.ScheduleParam{Type: "Manual"}
code, err := apiTest.AddScanAll(*admin, adminJob002)
require.NoError(suite.T(), err, "Error occurred while add a scan all job")
suite.Equal(201, code, "Add scan all status should be 200")
}
func (suite *ScanAllAPITestSuite) TestScanAllGet() {
apiTest := newHarborAPI()
code, _, err := apiTest.ScanAllScheduleGet(*admin)
require.NoError(suite.T(), err, "Error occurred while get a scan all job")
suite.Equal(200, code, "Get scan all status should be 200")
}

View File

@ -136,20 +136,19 @@ func (s *scheduler) schedule(ctx context.Context, vendorType string, vendorID in
CreationTime: now, CreationTime: now,
UpdateTime: now, UpdateTime: now,
} }
if params != nil {
paramsData, err := json.Marshal(params) paramsData, err := json.Marshal(params)
if err != nil { if err != nil {
return 0, err return 0, err
}
sched.CallbackFuncParam = string(paramsData)
} }
if extras != nil { sched.CallbackFuncParam = string(paramsData)
extrasData, err := json.Marshal(extras)
if err != nil { extrasData, err := json.Marshal(extras)
return 0, err if err != nil {
} return 0, err
sched.ExtraAttrs = string(extrasData)
} }
sched.ExtraAttrs = string(extrasData)
// create schedule record // create schedule record
// when checkin hook comes, the database record must exist, // when checkin hook comes, the database record must exist,
// so the database record must be created first before submitting job // so the database record must be created first before submitting job

View File

@ -4,18 +4,18 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"os"
"strings"
"github.com/go-openapi/runtime/middleware" "github.com/go-openapi/runtime/middleware"
"github.com/goharbor/harbor/src/controller/gc" "github.com/goharbor/harbor/src/controller/gc"
"github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task" "github.com/goharbor/harbor/src/pkg/task"
"github.com/goharbor/harbor/src/server/v2.0/handler/model" "github.com/goharbor/harbor/src/server/v2.0/handler/model"
"github.com/goharbor/harbor/src/server/v2.0/models" "github.com/goharbor/harbor/src/server/v2.0/models"
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/gc" operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/gc"
"os"
"strings"
) )
type gcAPI struct { type gcAPI struct {
@ -116,13 +116,13 @@ func (g *gcAPI) updateSchedule(ctx context.Context, cronType, cron string, polic
func (g *gcAPI) GetGCSchedule(ctx context.Context, params operation.GetGCScheduleParams) middleware.Responder { func (g *gcAPI) GetGCSchedule(ctx context.Context, params operation.GetGCScheduleParams) middleware.Responder {
schedule, err := g.gcCtr.GetSchedule(ctx) schedule, err := g.gcCtr.GetSchedule(ctx)
if errors.IsNotFoundErr(err) { if errors.IsNotFoundErr(err) {
return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(&scheduler.Schedule{}).ToSwagger()) return operation.NewGetGCScheduleOK()
} }
if err != nil { if err != nil {
return g.SendError(ctx, err) return g.SendError(ctx, err)
} }
return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(schedule).ToSwagger()) return operation.NewGetGCScheduleOK().WithPayload(model.NewGCSchedule(schedule).ToSwagger())
} }
func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryParams) middleware.Responder { func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryParams) middleware.Responder {
@ -171,7 +171,7 @@ func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryP
} }
func (g *gcAPI) GetGC(ctx context.Context, params operation.GetGCParams) middleware.Responder { func (g *gcAPI) GetGC(ctx context.Context, params operation.GetGCParams) middleware.Responder {
exec, err := g.gcCtr.GetExecution(ctx, params.GcID) exec, err := g.gcCtr.GetExecution(ctx, params.GCID)
if err != nil { if err != nil {
return g.SendError(ctx, err) return g.SendError(ctx, err)
} }
@ -199,13 +199,13 @@ func (g *gcAPI) GetGC(ctx context.Context, params operation.GetGCParams) middlew
func (g *gcAPI) GetGCLog(ctx context.Context, params operation.GetGCLogParams) middleware.Responder { func (g *gcAPI) GetGCLog(ctx context.Context, params operation.GetGCLogParams) middleware.Responder {
tasks, err := g.gcCtr.ListTasks(ctx, q.New(q.KeyWords{ tasks, err := g.gcCtr.ListTasks(ctx, q.New(q.KeyWords{
"ExecutionID": params.GcID, "ExecutionID": params.GCID,
})) }))
if err != nil { if err != nil {
return g.SendError(ctx, err) return g.SendError(ctx, err)
} }
if len(tasks) == 0 { if len(tasks) == 0 {
return g.SendError(ctx, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("garbage collection %d log is not found", params.GcID)) return g.SendError(ctx, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("garbage collection %d log is not found", params.GCID))
} }
log, err := g.gcCtr.GetTaskLog(ctx, tasks[0].ID) log, err := g.gcCtr.GetTaskLog(ctx, tasks[0].ID)
if err != nil { if err != nil {

View File

@ -19,7 +19,6 @@ import (
"net/http" "net/http"
lib_http "github.com/goharbor/harbor/src/lib/http" lib_http "github.com/goharbor/harbor/src/lib/http"
"github.com/goharbor/harbor/src/server/middleware" "github.com/goharbor/harbor/src/server/middleware"
"github.com/goharbor/harbor/src/server/middleware/blob" "github.com/goharbor/harbor/src/server/middleware/blob"
"github.com/goharbor/harbor/src/server/middleware/quota" "github.com/goharbor/harbor/src/server/middleware/quota"
@ -33,6 +32,7 @@ func New() http.Handler {
RepositoryAPI: newRepositoryAPI(), RepositoryAPI: newRepositoryAPI(),
AuditlogAPI: newAuditLogAPI(), AuditlogAPI: newAuditLogAPI(),
ScanAPI: newScanAPI(), ScanAPI: newScanAPI(),
ScanAllAPI: newScanAllAPI(),
ProjectAPI: newProjectAPI(), ProjectAPI: newProjectAPI(),
PreheatAPI: newPreheatAPI(), PreheatAPI: newPreheatAPI(),
IconAPI: newIconAPI(), IconAPI: newIconAPI(),
@ -41,7 +41,7 @@ func New() http.Handler {
ReplicationAPI: newReplicationAPI(), ReplicationAPI: newReplicationAPI(),
SysteminfoAPI: newSystemInfoAPI(), SysteminfoAPI: newSystemInfoAPI(),
PingAPI: newPingAPI(), PingAPI: newPingAPI(),
GcAPI: newGCAPI(), GCAPI: newGCAPI(),
}) })
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)

View File

@ -2,11 +2,13 @@ package model
import ( import (
"encoding/json" "encoding/json"
"strings"
"time"
"github.com/go-openapi/strfmt" "github.com/go-openapi/strfmt"
"github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/scheduler" "github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/server/v2.0/models" "github.com/goharbor/harbor/src/server/v2.0/models"
"time"
) )
// ScheduleParam defines the parameter of schedule trigger // ScheduleParam defines the parameter of schedule trigger
@ -41,44 +43,49 @@ func (h *GCHistory) ToSwagger() *models.GCHistory {
Deleted: h.Deleted, Deleted: h.Deleted,
JobStatus: h.Status, JobStatus: h.Status,
Schedule: &models.ScheduleObj{ Schedule: &models.ScheduleObj{
// covert MANUAL to Manual because the type of the ScheduleObj
// must be 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manual' and 'None'
Type: strings.Title(strings.ToLower(h.Schedule.Type)),
Cron: h.Schedule.Cron, Cron: h.Schedule.Cron,
Type: h.Schedule.Type,
}, },
CreationTime: strfmt.DateTime(h.CreationTime), CreationTime: strfmt.DateTime(h.CreationTime),
UpdateTime: strfmt.DateTime(h.UpdateTime), UpdateTime: strfmt.DateTime(h.UpdateTime),
} }
} }
// Schedule ... // GCSchedule ...
type Schedule struct { type GCSchedule struct {
*scheduler.Schedule *scheduler.Schedule
} }
// ToSwagger converts the schedule to the swagger model // ToSwagger converts the schedule to the swagger model
// TODO remove the hard code when after issue https://github.com/goharbor/harbor/issues/13047 is resolved. func (s *GCSchedule) ToSwagger() *models.GCHistory {
func (s *Schedule) ToSwagger() *models.GCHistory { if s.Schedule == nil {
return nil
}
e, err := json.Marshal(s.ExtraAttrs) e, err := json.Marshal(s.ExtraAttrs)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
} }
return &models.GCHistory{ return &models.GCHistory{
ID: 0, ID: s.ID,
JobName: "", JobName: "",
JobKind: s.CRON, JobKind: s.CRON,
JobParameters: string(e), JobParameters: string(e),
Deleted: false, Deleted: false,
JobStatus: "", JobStatus: s.Status,
Schedule: &models.ScheduleObj{ Schedule: &models.ScheduleObj{
Cron: s.CRON, Cron: s.CRON,
Type: "Custom", Type: s.CRONType,
}, },
CreationTime: strfmt.DateTime(s.CreationTime), CreationTime: strfmt.DateTime(s.CreationTime),
UpdateTime: strfmt.DateTime(s.UpdateTime), UpdateTime: strfmt.DateTime(s.UpdateTime),
} }
} }
// NewSchedule ... // NewGCSchedule ...
func NewSchedule(s *scheduler.Schedule) *Schedule { func NewGCSchedule(s *scheduler.Schedule) *GCSchedule {
return &Schedule{Schedule: s} return &GCSchedule{Schedule: s}
} }

View File

@ -0,0 +1,50 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
import (
"github.com/go-openapi/strfmt"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/server/v2.0/models"
)
// Schedule model
type Schedule struct {
*scheduler.Schedule
}
// ToSwagger converts the schedule to the swagger model
func (s *Schedule) ToSwagger() *models.Schedule {
if s.Schedule == nil {
return nil
}
return &models.Schedule{
ID: s.ID,
Status: s.Status,
Schedule: &models.ScheduleObj{
Cron: s.CRON,
Type: s.CRONType,
},
Parameters: s.ExtraAttrs,
CreationTime: strfmt.DateTime(s.CreationTime),
UpdateTime: strfmt.DateTime(s.UpdateTime),
}
}
// NewSchedule new schedule instance
func NewSchedule(schedule *scheduler.Schedule) *Schedule {
return &Schedule{Schedule: schedule}
}

View File

@ -0,0 +1,248 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package handler
import (
"context"
"fmt"
"github.com/go-openapi/runtime/middleware"
"github.com/goharbor/harbor/src/controller/scan"
"github.com/goharbor/harbor/src/controller/scanner"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
"github.com/goharbor/harbor/src/server/v2.0/handler/model"
"github.com/goharbor/harbor/src/server/v2.0/models"
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/scan_all"
)
func newScanAllAPI() *scanAllAPI {
return &scanAllAPI{
execMgr: task.ExecMgr,
scanCtl: scan.DefaultController,
scannerCtl: scanner.DefaultController,
scheduler: scheduler.Sched,
}
}
type scanAllAPI struct {
BaseAPI
execMgr task.ExecutionManager
scanCtl scan.Controller
scannerCtl scanner.Controller
scheduler scheduler.Scheduler
}
func (s *scanAllAPI) Prepare(ctx context.Context, operation string, params interface{}) middleware.Responder {
if err := s.RequireSysAdmin(ctx); err != nil {
return s.SendError(ctx, err)
}
if err := s.requireScanEnabled(ctx); err != nil {
return s.SendError(ctx, err)
}
return nil
}
func (s *scanAllAPI) CreateScanAllSchedule(ctx context.Context, params operation.CreateScanAllScheduleParams) middleware.Responder {
req := params.Schedule
if req.Schedule.Type == ScheduleNone {
return operation.NewCreateScanAllScheduleCreated()
}
if req.Schedule.Type == ScheduleManual {
execution, err := s.getLatestScanAllExecution(ctx, task.ExecutionTriggerManual)
if err != nil {
return s.SendError(ctx, err)
}
if execution != nil && execution.IsOnGoing() {
message := fmt.Sprintf("a previous scan all job aleady exits, its status is %s", execution.Status)
return s.SendError(ctx, errors.ConflictError(nil).WithMessage(message))
}
if _, err := s.scanCtl.ScanAll(ctx, task.ExecutionTriggerManual, true); err != nil {
return s.SendError(ctx, err)
}
} else {
schedule, err := s.getScanAllSchedule(ctx)
if err != nil {
return s.SendError(ctx, err)
}
if schedule != nil {
message := "fail to set schedule for scan all as always had one, please delete it firstly then to re-schedule"
return s.SendError(ctx, errors.PreconditionFailedError(nil).WithMessage(message))
}
if _, err := s.createOrUpdateScanAllSchedule(ctx, req.Schedule.Type, req.Schedule.Cron, nil); err != nil {
return s.SendError(ctx, err)
}
}
return operation.NewCreateScanAllScheduleCreated()
}
func (s *scanAllAPI) UpdateScanAllSchedule(ctx context.Context, params operation.UpdateScanAllScheduleParams) middleware.Responder {
req := params.Schedule
if req.Schedule.Type == ScheduleManual {
message := fmt.Sprintf("fail to update scan all schedule as wrong schedule type: %s", req.Schedule.Type)
return s.SendError(ctx, errors.BadRequestError(nil).WithMessage(message))
}
schedule, err := s.getScanAllSchedule(ctx)
if err != nil {
return s.SendError(ctx, err)
}
if req.Schedule.Type == ScheduleNone {
if schedule != nil {
err = s.scheduler.UnScheduleByID(ctx, schedule.ID)
}
} else {
_, err = s.createOrUpdateScanAllSchedule(ctx, req.Schedule.Type, req.Schedule.Cron, schedule)
}
if err != nil {
return s.SendError(ctx, err)
}
return operation.NewUpdateScanAllScheduleOK()
}
func (s *scanAllAPI) GetScanAllSchedule(ctx context.Context, params operation.GetScanAllScheduleParams) middleware.Responder {
schedule, err := s.getScanAllSchedule(ctx)
if err != nil {
return s.SendError(ctx, err)
}
return operation.NewGetScanAllScheduleOK().WithPayload(model.NewSchedule(schedule).ToSwagger())
}
func (s *scanAllAPI) GetLatestScanAllMetrics(ctx context.Context, params operation.GetLatestScanAllMetricsParams) middleware.Responder {
stats, err := s.getMetrics(ctx)
if err != nil {
return s.SendError(ctx, err)
}
return operation.NewGetLatestScanAllMetricsOK().WithPayload(stats)
}
func (s *scanAllAPI) GetLatestScheduledScanAllMetrics(ctx context.Context, params operation.GetLatestScheduledScanAllMetricsParams) middleware.Responder {
stats, err := s.getMetrics(ctx, task.ExecutionTriggerSchedule)
if err != nil {
return s.SendError(ctx, err)
}
return operation.NewGetLatestScanAllMetricsOK().WithPayload(stats)
}
func (s *scanAllAPI) createOrUpdateScanAllSchedule(ctx context.Context, cronType, cron string, previous *scheduler.Schedule) (int64, error) {
if previous != nil {
if cronType == previous.CRONType && cron == previous.CRON {
return previous.ID, nil
}
if err := s.scheduler.UnScheduleByID(ctx, previous.ID); err != nil {
return 0, err
}
}
return s.scheduler.Schedule(ctx, job.ImageScanAllJob, 0, cronType, cron, scan.ScanAllCallback, nil, nil)
}
func (s *scanAllAPI) getScanAllSchedule(ctx context.Context) (*scheduler.Schedule, error) {
query := q.New(q.KeyWords{"vendor_type": job.ImageScanAllJob})
schedules, err := s.scheduler.ListSchedules(ctx, query.First("-creation_time"))
if err != nil {
return nil, err
}
if len(schedules) > 1 {
return nil, fmt.Errorf("found more than one scheduled scan all job, please ensure that only one schedule left")
} else if len(schedules) == 0 {
return nil, nil
}
return schedules[0], nil
}
func (s *scanAllAPI) getMetrics(ctx context.Context, trigger ...string) (*models.Stats, error) {
execution, err := s.getLatestScanAllExecution(ctx, trigger...)
if err != nil {
return nil, err
}
sts := &models.Stats{}
if execution != nil && execution.Metrics != nil {
metrics := execution.Metrics
sts.Total = metrics.TaskCount
sts.Completed = metrics.SuccessTaskCount
sts.Metrics = map[string]int64{
"Pending": metrics.PendingTaskCount,
"Running": metrics.RunningTaskCount,
"Success": metrics.SuccessTaskCount,
"Error": metrics.ErrorTaskCount,
"Stopped": metrics.StoppedTaskCount,
}
sts.Ongoing = !job.Status(execution.Status).Final() || sts.Total != sts.Completed
}
return sts, nil
}
func (s *scanAllAPI) getLatestScanAllExecution(ctx context.Context, trigger ...string) (*task.Execution, error) {
query := q.New(q.KeyWords{"vendor_type": job.ImageScanAllJob})
if len(trigger) > 0 {
query.Keywords["trigger"] = trigger[0]
}
executions, err := s.execMgr.List(ctx, query.First("-start_time"))
if err != nil {
return nil, err
}
if len(executions) == 0 {
return nil, nil
}
return executions[0], nil
}
func (s *scanAllAPI) requireScanEnabled(ctx context.Context) error {
kws := make(map[string]interface{})
kws["is_default"] = true
query := &q.Query{
Keywords: kws,
}
l, err := s.scannerCtl.ListRegistrations(query)
if err != nil {
return errors.Wrap(err, "check if scan is enabled")
}
if len(l) == 0 {
return errors.PreconditionFailedError(nil).WithMessage("no scanner is configured, it's not possible to scan")
}
return nil
}

View File

@ -0,0 +1,520 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package handler
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"testing"
"time"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
taskdao "github.com/goharbor/harbor/src/pkg/task/dao"
"github.com/goharbor/harbor/src/server/v2.0/models"
"github.com/goharbor/harbor/src/server/v2.0/restapi"
scantesting "github.com/goharbor/harbor/src/testing/controller/scan"
scannertesting "github.com/goharbor/harbor/src/testing/controller/scanner"
"github.com/goharbor/harbor/src/testing/mock"
schedulertesting "github.com/goharbor/harbor/src/testing/pkg/scheduler"
tasktesting "github.com/goharbor/harbor/src/testing/pkg/task"
htesting "github.com/goharbor/harbor/src/testing/server/v2.0/handler"
"github.com/stretchr/testify/suite"
)
type ScanAllTestSuite struct {
htesting.Suite
execMgr *tasktesting.ExecutionManager
scanCtl *scantesting.Controller
scannerCtl *scannertesting.Controller
scheduler *schedulertesting.Scheduler
execution *task.Execution
schedule *scheduler.Schedule
}
func (suite *ScanAllTestSuite) SetupSuite() {
suite.execution = &task.Execution{
Status: "Running",
Metrics: &taskdao.Metrics{
TaskCount: 10,
SuccessTaskCount: 5,
ErrorTaskCount: 0,
PendingTaskCount: 4,
RunningTaskCount: 1,
ScheduledTaskCount: 0,
StoppedTaskCount: 0,
},
}
suite.schedule = &scheduler.Schedule{
ID: 1,
VendorType: "vendor_type",
CRONType: "Daily",
CRON: "0 0 0 * * *",
Status: "Running",
CreationTime: time.Now(),
UpdateTime: time.Now(),
}
suite.execMgr = &tasktesting.ExecutionManager{}
suite.scanCtl = &scantesting.Controller{}
suite.scannerCtl = &scannertesting.Controller{}
suite.scheduler = &schedulertesting.Scheduler{}
suite.Config = &restapi.Config{
ScanAllAPI: &scanAllAPI{
execMgr: suite.execMgr,
scanCtl: suite.scanCtl,
scannerCtl: suite.scannerCtl,
scheduler: suite.scheduler,
},
}
suite.Suite.SetupSuite()
}
func (suite *ScanAllTestSuite) TestAuthorization() {
newBody := func(body interface{}) io.Reader {
if body == nil {
return nil
}
buf, err := json.Marshal(body)
suite.Require().NoError(err)
return bytes.NewBuffer(buf)
}
schedule := models.Schedule{
Schedule: &models.ScheduleObj{Type: "Manual"},
}
reqs := []struct {
method string
url string
body interface{}
}{
{http.MethodGet, "/scans/all/metrics", nil},
{http.MethodGet, "/scans/schedule/metrics", nil},
{http.MethodGet, "/system/scanAll/schedule", nil},
{http.MethodPut, "/system/scanAll/schedule", schedule},
{http.MethodPost, "/system/scanAll/schedule", schedule},
}
for _, req := range reqs {
{
// authorized required
suite.Security.On("IsAuthenticated").Return(false).Once()
res, err := suite.DoReq(req.method, req.url, newBody(req.body))
suite.NoError(err)
suite.Equal(401, res.StatusCode)
}
{
// system admin required
suite.Security.On("IsAuthenticated").Return(true).Once()
suite.Security.On("IsSysAdmin").Return(false).Once()
suite.Security.On("GetUsername").Return("username").Once()
res, err := suite.DoReq(req.method, req.url, newBody(req.body))
suite.NoError(err)
suite.Equal(403, res.StatusCode)
}
{
// default scanner required
suite.Security.On("IsAuthenticated").Return(true).Once()
suite.Security.On("IsSysAdmin").Return(true).Once()
mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return(nil, nil).Once()
res, err := suite.DoReq(req.method, req.url, newBody(req.body))
suite.NoError(err)
suite.Equal(412, res.StatusCode)
}
{
// default scanner required failed
suite.Security.On("IsAuthenticated").Return(true).Once()
suite.Security.On("IsSysAdmin").Return(true).Once()
mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return(nil, fmt.Errorf("failed")).Once()
res, err := suite.DoReq(req.method, req.url, newBody(req.body))
suite.NoError(err)
suite.Equal(500, res.StatusCode)
}
}
}
func (suite *ScanAllTestSuite) TestGetLatestScanAllMetrics() {
times := 3
suite.Security.On("IsAuthenticated").Return(true).Times(times)
suite.Security.On("IsSysAdmin").Return(true).Times(times)
mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times)
{
// get scan all execution failed
mock.OnAnything(suite.execMgr, "List").Return(nil, fmt.Errorf("failed to list executions")).Once()
res, err := suite.Get("/scans/all/metrics")
suite.NoError(err)
suite.Equal(500, res.StatusCode)
}
{
// scan all execution not found
mock.OnAnything(suite.execMgr, "List").Return(nil, nil).Once()
var stats map[string]interface{}
res, err := suite.GetJSON("/scans/all/metrics", &stats)
suite.NoError(err)
suite.Equal(200, res.StatusCode)
suite.Contains(stats, "ongoing")
}
{
// scan all execution found
mock.OnAnything(suite.execMgr, "List").Return([]*task.Execution{suite.execution}, nil).Once()
var stats models.Stats
res, err := suite.GetJSON("/scans/all/metrics", &stats)
suite.NoError(err)
suite.Equal(200, res.StatusCode)
suite.True(stats.Ongoing)
}
}
func (suite *ScanAllTestSuite) TestGetLatestScheduledScanAllMetrics() {
times := 3
suite.Security.On("IsAuthenticated").Return(true).Times(times)
suite.Security.On("IsSysAdmin").Return(true).Times(times)
mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times)
{
// get scan all execution failed
mock.OnAnything(suite.execMgr, "List").Return(nil, fmt.Errorf("failed to list executions")).Once()
res, err := suite.Get("/scans/schedule/metrics")
suite.NoError(err)
suite.Equal(500, res.StatusCode)
}
{
// scan all execution not found
mock.OnAnything(suite.execMgr, "List").Return(nil, nil).Once()
var stats map[string]interface{}
res, err := suite.GetJSON("/scans/schedule/metrics", &stats)
suite.NoError(err)
suite.Equal(200, res.StatusCode)
suite.Contains(stats, "ongoing")
}
{
// scan all execution found
mock.OnAnything(suite.execMgr, "List").Return([]*task.Execution{suite.execution}, nil).Once()
var stats models.Stats
res, err := suite.GetJSON("/scans/schedule/metrics", &stats)
suite.NoError(err)
suite.Equal(200, res.StatusCode)
suite.True(stats.Ongoing)
}
}
func (suite *ScanAllTestSuite) TestCreateScanAllSchedule() {
times := 11
suite.Security.On("IsAuthenticated").Return(true).Times(times)
suite.Security.On("IsSysAdmin").Return(true).Times(times)
mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times)
{
// create scan all schedule no body
res, err := suite.Post("/system/scanAll/schedule", nil)
suite.NoError(err)
suite.Equal(422, res.StatusCode)
}
{
// create scan all schedule with bad body
res, err := suite.Post("/system/scanAll/schedule", bytes.NewBuffer([]byte("bad body")))
suite.NoError(err)
suite.Equal(422, res.StatusCode)
}
{
// create scan all schedule with ScheduleNone
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}}
res, err := suite.PostJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(201, res.StatusCode)
}
{
// create scan all schedule with ScheduleManual but get latest scan all execution failed
mock.OnAnything(suite.execMgr, "List").Return(nil, fmt.Errorf("list executions failed")).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}}
res, err := suite.PostJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(500, res.StatusCode)
}
{
// create scan all schedule with ScheduleManual but a previous scan all job aleady exits
mock.OnAnything(suite.execMgr, "List").Return([]*task.Execution{suite.execution}, nil).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}}
res, err := suite.PostJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(409, res.StatusCode)
}
{
// create scan all schedule with ScheduleManual no previous scan all job exits
mock.OnAnything(suite.execMgr, "List").Return(nil, nil).Once()
mock.OnAnything(suite.scanCtl, "ScanAll").Return(int64(1), nil).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}}
res, err := suite.PostJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(201, res.StatusCode)
}
{
// create scan all schedule with ScheduleManual but scan all failed
mock.OnAnything(suite.execMgr, "List").Return(nil, nil).Once()
mock.OnAnything(suite.scanCtl, "ScanAll").Return(int64(0), fmt.Errorf("scan all failed")).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}}
res, err := suite.PostJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(500, res.StatusCode)
}
{
// create scan all schedule with periodic but get latest schedule failed
mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, fmt.Errorf("get schedule failed")).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}}
res, err := suite.PostJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(500, res.StatusCode)
}
{
// create scan all schedule with periodic but schedule areadly exists
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}}
res, err := suite.PostJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(412, res.StatusCode)
}
{
// create scan all schedule with periodic but create schedule failed
mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, nil).Once()
mock.OnAnything(suite.scheduler, "Schedule").Return(int64(0), fmt.Errorf("create schedule failed")).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}}
res, err := suite.PostJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(500, res.StatusCode)
}
{
// create scan all schedule with periodic
mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, nil).Once()
mock.OnAnything(suite.scheduler, "Schedule").Return(int64(1), nil).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}}
res, err := suite.PostJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(201, res.StatusCode)
}
}
func (suite *ScanAllTestSuite) TestUpdateScanAllSchedule() {
times := 11
suite.Security.On("IsAuthenticated").Return(true).Times(times)
suite.Security.On("IsSysAdmin").Return(true).Times(times)
mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times)
{
// update scan all schedule no body
res, err := suite.Put("/system/scanAll/schedule", nil)
suite.NoError(err)
suite.Equal(422, res.StatusCode)
}
{
// update scan all schedule with bad body
res, err := suite.Put("/system/scanAll/schedule", bytes.NewBuffer([]byte("bad body")))
suite.NoError(err)
suite.Equal(422, res.StatusCode)
}
{
// update scan all schedule with ScheduleManual
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}}
res, err := suite.PutJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(400, res.StatusCode)
}
{
// update scan all schedule but get schedule failed
mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, fmt.Errorf("get schedule failed")).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}}
res, err := suite.PutJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(500, res.StatusCode)
}
{
// update scan all schedule with ScheduleNone when no schedule found
mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, nil).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}}
res, err := suite.PutJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(200, res.StatusCode)
}
{
// update scan all schedule with ScheduleNone and unschedule failed
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(fmt.Errorf("unschedule failed")).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}}
res, err := suite.PutJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(500, res.StatusCode)
}
{
// update scan all schedule with ScheduleNone successfully
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(nil).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}}
res, err := suite.PutJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(200, res.StatusCode)
}
{
// update scan all schedule with periodic but schedule not changed
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}}
res, err := suite.PutJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(200, res.StatusCode)
}
{
// update scan all schedule with periodic and schedule changed
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(nil).Once()
mock.OnAnything(suite.scheduler, "Schedule").Return(int64(1), nil).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleCustom, Cron: "0 1 0 * * *"}}
res, err := suite.PutJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(200, res.StatusCode)
}
{
// update scan all schedule with periodic and schedule changed, but unschedule old schedule failed
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(fmt.Errorf("unschedule failed")).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleCustom, Cron: "0 1 0 * * *"}}
res, err := suite.PutJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(500, res.StatusCode)
}
{
// update scan all schedule with periodic and schedule changed, but creat new schedule failed
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(nil).Once()
mock.OnAnything(suite.scheduler, "Schedule").Return(int64(0), fmt.Errorf("create schedule failed")).Once()
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleCustom, Cron: "0 1 0 * * *"}}
res, err := suite.PutJSON("/system/scanAll/schedule", body)
suite.NoError(err)
suite.Equal(500, res.StatusCode)
}
}
func (suite *ScanAllTestSuite) TestGetScanAllSchedule() {
times := 4
suite.Security.On("IsAuthenticated").Return(true).Times(times)
suite.Security.On("IsSysAdmin").Return(true).Times(times)
mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times)
{
// get schedule failed
mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, fmt.Errorf("get schedule failed")).Once()
res, err := suite.Get("/system/scanAll/schedule")
suite.NoError(err)
suite.Equal(500, res.StatusCode)
}
{
// schedule not found
mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, nil).Once()
res, err := suite.Get("/system/scanAll/schedule")
suite.NoError(err)
suite.Equal(200, res.StatusCode)
}
{
// schedule found
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
var schedule models.Schedule
res, err := suite.GetJSON("/system/scanAll/schedule", &schedule)
suite.NoError(err)
suite.Equal(200, res.StatusCode)
suite.Equal(suite.schedule.CRONType, schedule.Schedule.Type)
suite.Equal(suite.schedule.CRON, schedule.Schedule.Cron)
}
{
// schedule found more than one
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule, suite.schedule}, nil).Once()
res, err := suite.Get("/system/scanAll/schedule")
suite.NoError(err)
suite.Equal(500, res.StatusCode)
}
}
func TestScanAllTestSuite(t *testing.T) {
suite.Run(t, &ScanAllTestSuite{})
}

View File

@ -45,7 +45,6 @@ func registerLegacyRoutes() {
beego.Router("/api/"+version+"/quotas", &api.QuotaAPI{}, "get:List") beego.Router("/api/"+version+"/quotas", &api.QuotaAPI{}, "get:List")
beego.Router("/api/"+version+"/quotas/:id([0-9]+)", &api.QuotaAPI{}, "get:Get;put:Put") beego.Router("/api/"+version+"/quotas/:id([0-9]+)", &api.QuotaAPI{}, "get:Get;put:Put")
beego.Router("/api/"+version+"/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/"+version+"/system/CVEAllowlist", &api.SysCVEAllowlistAPI{}, "get:Get;put:Put") beego.Router("/api/"+version+"/system/CVEAllowlist", &api.SysCVEAllowlistAPI{}, "get:Get;put:Put")
beego.Router("/api/"+version+"/system/oidc/ping", &api.OIDCAPI{}, "post:Ping") beego.Router("/api/"+version+"/system/oidc/ping", &api.OIDCAPI{}, "post:Ping")
@ -106,9 +105,4 @@ func registerLegacyRoutes() {
proScannerAPI := &api.ProjectScannerAPI{} proScannerAPI := &api.ProjectScannerAPI{}
beego.Router("/api/"+version+"/projects/:pid([0-9]+)/scanner", proScannerAPI, "get:GetProjectScanner;put:SetProjectScanner") beego.Router("/api/"+version+"/projects/:pid([0-9]+)/scanner", proScannerAPI, "get:GetProjectScanner;put:SetProjectScanner")
beego.Router("/api/"+version+"/projects/:pid([0-9]+)/scanner/candidates", proScannerAPI, "get:GetProScannerCandidates") beego.Router("/api/"+version+"/projects/:pid([0-9]+)/scanner/candidates", proScannerAPI, "get:GetProScannerCandidates")
// Add routes for scan all metrics
scanAllAPI := &api.ScanAllAPI{}
beego.Router("/api/"+version+"/scans/all/metrics", scanAllAPI, "get:GetScanAllMetrics")
beego.Router("/api/"+version+"/scans/schedule/metrics", scanAllAPI, "get:GetScheduleMetrics")
} }

View File

@ -0,0 +1,158 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package handler
import (
"bytes"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"github.com/goharbor/harbor/src/common/security"
lib "github.com/goharbor/harbor/src/lib/http"
"github.com/goharbor/harbor/src/server/middleware"
"github.com/goharbor/harbor/src/server/v2.0/restapi"
securitytesting "github.com/goharbor/harbor/src/testing/common/security"
"github.com/stretchr/testify/suite"
)
// Suite ...
type Suite struct {
suite.Suite
Config *restapi.Config
Security *securitytesting.Context
ts *httptest.Server
tc *http.Client
}
// SetupSuite ...
func (suite *Suite) SetupSuite() {
h, api, _ := restapi.HandlerAPI(*suite.Config)
api.ServeError = func(rw http.ResponseWriter, r *http.Request, err error) {
lib.SendError(rw, err)
}
suite.Security = &securitytesting.Context{}
m := middleware.New(func(w http.ResponseWriter, r *http.Request, next http.Handler) {
next.ServeHTTP(w, r.WithContext(security.NewContext(r.Context(), suite.Security)))
})
suite.ts = httptest.NewServer(m(h))
suite.tc = http.DefaultClient
}
// TearDownSuite ...
func (suite *Suite) TearDownSuite() {
suite.ts.Close()
}
// DoReq ...
func (suite *Suite) DoReq(method string, url string, body io.Reader, contentTypes ...string) (*http.Response, error) {
req, err := http.NewRequest(method, suite.ts.URL+"/api/v2.0"+url, body)
if err != nil {
return nil, err
}
contentType := "application/json"
if len(contentTypes) > 0 {
contentType = contentTypes[0]
}
req.Header.Set("Content-Type", contentType)
return suite.tc.Do(req)
}
// Delete ...
func (suite *Suite) Delete(url string, contentTypes ...string) (*http.Response, error) {
return suite.DoReq(http.MethodDelete, url, nil, contentTypes...)
}
// Get ...
func (suite *Suite) Get(url string, contentTypes ...string) (*http.Response, error) {
return suite.DoReq(http.MethodGet, url, nil, contentTypes...)
}
// GetJSON ...
func (suite *Suite) GetJSON(url string, js interface{}) (*http.Response, error) {
res, err := suite.Get(url)
if err != nil {
return nil, err
}
if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest {
data, err := ioutil.ReadAll(res.Body)
if err != nil {
return res, err
}
res.Body.Close()
if err := json.Unmarshal(data, js); err != nil {
return res, err
}
res.Body = ioutil.NopCloser(bytes.NewBuffer(data))
}
return res, nil
}
// Patch ...
func (suite *Suite) Patch(url string, body io.Reader, contentTypes ...string) (*http.Response, error) {
return suite.DoReq(http.MethodPatch, url, body, contentTypes...)
}
// PatchJSON ...
func (suite *Suite) PatchJSON(url string, js interface{}) (*http.Response, error) {
buf, err := json.Marshal(js)
if err != nil {
return nil, err
}
return suite.Patch(url, bytes.NewBuffer(buf))
}
// Post ...
func (suite *Suite) Post(url string, body io.Reader, contentTypes ...string) (*http.Response, error) {
return suite.DoReq(http.MethodPost, url, body, contentTypes...)
}
// PostJSON ...
func (suite *Suite) PostJSON(url string, js interface{}) (*http.Response, error) {
buf, err := json.Marshal(js)
if err != nil {
return nil, err
}
return suite.Post(url, bytes.NewBuffer(buf))
}
// Put ...
func (suite *Suite) Put(url string, body io.Reader, contentTypes ...string) (*http.Response, error) {
return suite.DoReq(http.MethodPut, url, body, contentTypes...)
}
// PutJSON ...
func (suite *Suite) PutJSON(url string, js interface{}) (*http.Response, error) {
buf, err := json.Marshal(js)
if err != nil {
return nil, err
}
return suite.Put(url, bytes.NewBuffer(buf))
}

View File

@ -28,7 +28,7 @@ def get_endpoint():
def _create_client(server, credential, debug, api_type="products"): def _create_client(server, credential, debug, api_type="products"):
cfg = None cfg = None
if api_type in ('projectv2', 'artifact', 'repository', 'scan', 'preheat', 'replication', 'robot', 'gc'): if api_type in ('projectv2', 'artifact', 'repository', 'scan', 'scanall', 'preheat', 'replication', 'robot', 'gc'):
cfg = v2_swagger_client.Configuration() cfg = v2_swagger_client.Configuration()
else: else:
cfg = swagger_client.Configuration() cfg = swagger_client.Configuration()
@ -58,6 +58,7 @@ def _create_client(server, credential, debug, api_type="products"):
"preheat": v2_swagger_client.PreheatApi(v2_swagger_client.ApiClient(cfg)), "preheat": v2_swagger_client.PreheatApi(v2_swagger_client.ApiClient(cfg)),
"repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)), "repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)),
"scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)), "scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)),
"scanall": v2_swagger_client.ScanAllApi(v2_swagger_client.ApiClient(cfg)),
"scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)), "scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)),
"replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)), "replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)),
"robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)), "robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)),

View File

@ -0,0 +1,50 @@
# -*- coding: utf-8 -*-
import time
import base
import v2_swagger_client
from v2_swagger_client.rest import ApiException
class ScanAll(base.Base):
def __init__(self):
super(ScanAll,self).__init__(api_type="scanall")
def create_scan_all_schedule(self, schedule_type, cron=None, expect_status_code=201, expect_response_body=None, **kwargs):
client = self._get_client(**kwargs)
schedule_obj = v2_swagger_client.ScheduleObj()
schedule_obj.type = schedule_type
if cron is not None:
schedule_obj.cron = cron
schedule = v2_swagger_client.Schedule()
schedule.schedule = schedule_obj
try:
_, status_code, _ = client.create_scan_all_schedule_with_http_info(schedule)
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Create scan all schedule response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Create scan all schedule result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
def scan_all_now(self, **kwargs):
self.create_scan_all_schedule('Manual', **kwargs)
def wait_until_scans_all_finish(self, **kwargs):
client = self._get_client(**kwargs)
timeout_count = 50
while True:
time.sleep(5)
timeout_count = timeout_count - 1
if (timeout_count == 0):
break
stats = client.get_latest_scan_all_metrics()
print("Scan all status:", stats)
if stats.ongoing is False:
return
raise Exception("Error: Scan all job is timeout.")

View File

@ -117,46 +117,6 @@ class System(base.Base):
base._assert_status_code(expect_status_code, status_code) base._assert_status_code(expect_status_code, status_code)
return base._get_id_from_header(header) return base._get_id_from_header(header)
def wait_until_scans_all_finish(self, **kwargs):
client = self._get_client(**kwargs)
timeout_count = 50
scan_status=""
while True:
time.sleep(5)
timeout_count = timeout_count - 1
if (timeout_count == 0):
break
stats = client.scans_all_metrics_get()
print("Scan all status:", stats)
if stats.ongoing is False:
return
raise Exception("Error: Scan all job is timeout.")
def create_scan_all_schedule(self, schedule_type, cron = None, expect_status_code = 201, expect_response_body = None, **kwargs):
client = self._get_client(**kwargs)
scanschedule = swagger_client.AdminJobScheduleObj()
scanschedule.type = schedule_type
if cron is not None:
scanschedule.cron = cron
scan_all_schedule = swagger_client.AdminJobSchedule(scanschedule)
try:
_, status_code, header = client.system_scan_all_schedule_post_with_http_info(scan_all_schedule)
except ApiException as e:
if e.status == expect_status_code:
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
raise Exception(r"Create Scan All schedule response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
else:
return e.reason, e.body
else:
raise Exception(r"Create Scan All schedule result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
base._assert_status_code(expect_status_code, status_code)
return base._get_id_from_header(header)
def scan_now(self, **kwargs):
scan_all_id = self.create_scan_all_schedule('Manual', **kwargs)
return scan_all_id
def set_cve_allowlist(self, expires_at=None, expected_status_code=200, *cve_ids, **kwargs): def set_cve_allowlist(self, expires_at=None, expected_status_code=200, *cve_ids, **kwargs):
client = self._get_client(**kwargs) client = self._get_client(**kwargs)
cve_list = [swagger_client.CVEAllowlistItem(cve_id=c) for c in cve_ids] cve_list = [swagger_client.CVEAllowlistItem(cve_id=c) for c in cve_ids]

View File

@ -4,20 +4,21 @@ import unittest
from testutils import harbor_server, suppress_urllib3_warning from testutils import harbor_server, suppress_urllib3_warning
from testutils import TEARDOWN from testutils import TEARDOWN
from testutils import ADMIN_CLIENT from testutils import ADMIN_CLIENT
from library.system import System
from library.project import Project from library.project import Project
from library.user import User from library.user import User
from library.repository import Repository from library.repository import Repository
from library.repository import push_self_build_image_to_project from library.repository import push_self_build_image_to_project
from library.artifact import Artifact from library.artifact import Artifact
from library.scan_all import ScanAll
class TestScanAll(unittest.TestCase): class TestScanAll(unittest.TestCase):
@suppress_urllib3_warning @suppress_urllib3_warning
def setUp(self): def setUp(self):
self.system = System()
self.project= Project() self.project= Project()
self.user= User() self.user= User()
self.artifact = Artifact() self.artifact = Artifact()
self.repo = Repository() self.repo = Repository()
self.scan_all = ScanAll()
@unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
def tearDown(self): def tearDown(self):
@ -81,8 +82,8 @@ class TestScanAll(unittest.TestCase):
TestScanAll.repo_Luca_name, tag_Luca = push_self_build_image_to_project(TestScanAll.project_Luca_name, harbor_server, user_Luca_name, user_common_password, image_b, src_tag) TestScanAll.repo_Luca_name, tag_Luca = push_self_build_image_to_project(TestScanAll.project_Luca_name, harbor_server, user_Luca_name, user_common_password, image_b, src_tag)
#4. Trigger scan all event; #4. Trigger scan all event;
self.system.scan_now(**ADMIN_CLIENT) self.scan_all.scan_all_now(**ADMIN_CLIENT)
self.system.wait_until_scans_all_finish(**ADMIN_CLIENT) self.scan_all.wait_until_scans_all_finish(**ADMIN_CLIENT)
#5. Check if image in project_Alice and another image in project_Luca were both scanned. #5. Check if image in project_Alice and another image in project_Luca were both scanned.
self.artifact.check_image_scan_result(TestScanAll.project_Alice_name, image_a, tag_Alice, **USER_ALICE_CLIENT) self.artifact.check_image_scan_result(TestScanAll.project_Alice_name, image_a, tag_Alice, **USER_ALICE_CLIENT)