mirror of https://github.com/goharbor/harbor.git
Merge pull request #13813 from heww/scan-all-apis
refactor(api): move scan all apis to go-swagger
This commit is contained in:
Wenkai Yin(尹文开)
GitHub
commit
53c8ad8228
2
Makefile
2
Makefile
|
|
@ -296,7 +296,7 @@ endif
|
|||
SWAGGER_IMAGENAME=goharbor/swagger
|
||||
SWAGGER_VERSION=v0.21.0
|
||||
SWAGGER=$(DOCKERCMD) run --rm -u $(shell id -u):$(shell id -g) -v $(BUILDPATH):$(BUILDPATH) -w $(BUILDPATH) ${SWAGGER_IMAGENAME}:${SWAGGER_VERSION}
|
||||
SWAGGER_GENERATE_SERVER=${SWAGGER} generate server --template-dir=$(TOOLSPATH)/swagger/templates --exclude-main --additional-initialism=CVE
|
||||
SWAGGER_GENERATE_SERVER=${SWAGGER} generate server --template-dir=$(TOOLSPATH)/swagger/templates --exclude-main --additional-initialism=CVE --additional-initialism=GC
|
||||
SWAGGER_IMAGE_BUILD_CMD=${DOCKERBUILD} -f ${TOOLSPATH}/swagger/Dockerfile --build-arg SWAGGER_VERSION=${SWAGGER_VERSION} -t ${SWAGGER_IMAGENAME}:$(SWAGGER_VERSION) .
|
||||
|
||||
SWAGGER_IMAGENAME:
|
||||
|
|
|
|||
|
|
@ -1549,99 +1549,6 @@ paths:
|
|||
description: Only admin has this authority.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
'/system/gc/{id}':
|
||||
get:
|
||||
summary: Get gc status.
|
||||
description: This endpoint let user get gc status filtered by specific ID.
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
type: integer
|
||||
format: int64
|
||||
required: true
|
||||
description: Relevant job ID
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'200':
|
||||
description: Get gc results successfully.
|
||||
schema:
|
||||
$ref: '#/definitions/GCResult'
|
||||
'401':
|
||||
description: User need to log in first.
|
||||
'403':
|
||||
description: User does not have permission of admin role.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
/system/scanAll/schedule:
|
||||
get:
|
||||
summary: Get scan_all's schedule.
|
||||
description: This endpoint is for getting a schedule for the scan all job, which scans all of images in Harbor.
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'200':
|
||||
description: Get a schedule for the scan all job, which scans all of images in Harbor.
|
||||
schema:
|
||||
$ref: '#/definitions/AdminJobSchedule'
|
||||
'401':
|
||||
description: User need to log in first.
|
||||
'403':
|
||||
description: Only admin has this authority.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
put:
|
||||
summary: Update scan all's schedule.
|
||||
description: |
|
||||
This endpoint is for updating the schedule of scan all job, which scans all of images in Harbor.
|
||||
parameters:
|
||||
- name: schedule
|
||||
in: body
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/AdminJobSchedule'
|
||||
description: Updates the schedule of scan all job, which scans all of images in Harbor.
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'200':
|
||||
description: Updated scan_all's schedule successfully.
|
||||
'400':
|
||||
description: Invalid schedule type.
|
||||
'401':
|
||||
description: User need to log in first.
|
||||
'403':
|
||||
description: User does not have permission of admin role.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
post:
|
||||
summary: Create a schedule or a manual trigger for the scan all job.
|
||||
description: |
|
||||
This endpoint is for creating a schedule or a manual trigger for the scan all job, which scans all of images in Harbor.
|
||||
parameters:
|
||||
- name: schedule
|
||||
in: body
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/AdminJobSchedule'
|
||||
description: Create a schedule or a manual trigger for the scan all job.
|
||||
tags:
|
||||
- Products
|
||||
responses:
|
||||
'200':
|
||||
description: Updated scan_all's schedule successfully.
|
||||
'400':
|
||||
description: Invalid schedule type.
|
||||
'401':
|
||||
description: User need to log in first.
|
||||
'403':
|
||||
description: User does not have permission of admin role.
|
||||
'409':
|
||||
description: There is a "scanall" job in progress, so the request cannot be served.
|
||||
'500':
|
||||
description: Unexpected internal errors.
|
||||
'503':
|
||||
description: Harbor is not deployed with scanners.
|
||||
/configurations:
|
||||
get:
|
||||
summary: Get system configurations.
|
||||
|
|
@ -3074,42 +2981,6 @@ paths:
|
|||
description: Request is not allowed
|
||||
'500':
|
||||
description: Internal server error happened
|
||||
'/scans/all/metrics':
|
||||
get:
|
||||
summary: Get the metrics of the latest scan all process
|
||||
description: Get the metrics of the latest scan all process
|
||||
tags:
|
||||
- Products
|
||||
- Scan
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
schema:
|
||||
$ref: '#/definitions/Stats'
|
||||
'401':
|
||||
description: Unauthorized request
|
||||
'403':
|
||||
description: Request is not allowed
|
||||
'500':
|
||||
description: Internal server error happened
|
||||
'/scans/schedule/metrics':
|
||||
get:
|
||||
summary: Get the metrics of the latest scheduled scan all process
|
||||
description: Get the metrics of the latest scheduled scan all process
|
||||
tags:
|
||||
- Products
|
||||
- Scan
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
schema:
|
||||
$ref: '#/definitions/Stats'
|
||||
'401':
|
||||
description: Unauthorized request
|
||||
'403':
|
||||
description: Request is not allowed
|
||||
'500':
|
||||
description: Internal server error happened
|
||||
responses:
|
||||
OK:
|
||||
description: 'Success'
|
||||
|
|
@ -4152,56 +4023,6 @@ definitions:
|
|||
properties:
|
||||
labels:
|
||||
$ref: '#/definitions/Labels'
|
||||
GCResult:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
type: integer
|
||||
description: the id of gc job.
|
||||
job_name:
|
||||
type: string
|
||||
description: the job name of gc job.
|
||||
job_kind:
|
||||
type: string
|
||||
description: the job kind of gc job.
|
||||
job_parameters:
|
||||
type: string
|
||||
description: the job parameters of gc job.
|
||||
schedule:
|
||||
$ref: '#/definitions/AdminJobScheduleObj'
|
||||
job_status:
|
||||
type: string
|
||||
description: the status of gc job.
|
||||
deleted:
|
||||
type: boolean
|
||||
description: if gc job was deleted.
|
||||
creation_time:
|
||||
type: string
|
||||
description: the creation time of gc job.
|
||||
update_time:
|
||||
type: string
|
||||
description: the update time of gc job.
|
||||
AdminJobSchedule:
|
||||
type: object
|
||||
properties:
|
||||
schedule:
|
||||
$ref: '#/definitions/AdminJobScheduleObj'
|
||||
parameters:
|
||||
type: object
|
||||
description: The parameters of admin job
|
||||
additionalProperties:
|
||||
type: boolean
|
||||
AdminJobScheduleObj:
|
||||
type: object
|
||||
properties:
|
||||
type:
|
||||
type: string
|
||||
description: |
|
||||
The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manually' and 'None'.
|
||||
'Manually' means to trigger it right away and 'None' means to cancel the schedule.
|
||||
cron:
|
||||
type: string
|
||||
description: A cron expression, a time-based job scheduler.
|
||||
SearchResult:
|
||||
type: object
|
||||
description: The chart search result item
|
||||
|
|
@ -4848,39 +4669,6 @@ definitions:
|
|||
type: string
|
||||
description: The identifier of the scanner registration
|
||||
|
||||
Stats:
|
||||
type: object
|
||||
description: Stats provides the overall progress of the scan all process.
|
||||
properties:
|
||||
total:
|
||||
type: integer
|
||||
format: int
|
||||
description: 'The total number of scan processes triggered by the scan all action'
|
||||
example: 100
|
||||
completed:
|
||||
type: integer
|
||||
format: int
|
||||
description: 'The number of the finished scan processes triggered by the scan all action'
|
||||
example: 90
|
||||
requester:
|
||||
type: string
|
||||
description: 'The requester identity which usually uses the ID of the scan all job'
|
||||
example: '28'
|
||||
metrics:
|
||||
type: object
|
||||
description: 'The metrics data for the each status'
|
||||
additionalProperties:
|
||||
type: integer
|
||||
format: int
|
||||
example: 10
|
||||
example:
|
||||
'Success': 5
|
||||
'Error': 2,
|
||||
'Running': 3
|
||||
ongoing:
|
||||
type: boolean
|
||||
description: A flag indicating job status of scan all .
|
||||
|
||||
SupportedWebhookEventTypes:
|
||||
type: object
|
||||
description: Supportted webhook event types and notify types.
|
||||
|
|
|
|||
|
|
@ -1976,6 +1976,47 @@ paths:
|
|||
$ref: '#/responses/404'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
/scans/all/metrics:
|
||||
get:
|
||||
summary: Get the metrics of the latest scan all process
|
||||
description: Get the metrics of the latest scan all process
|
||||
tags:
|
||||
- scanAll
|
||||
operationId: getLatestScanAllMetrics
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
schema:
|
||||
$ref: '#/definitions/Stats'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'412':
|
||||
$ref: '#/responses/412'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
/scans/schedule/metrics:
|
||||
get:
|
||||
summary: Get the metrics of the latest scheduled scan all process
|
||||
description: Get the metrics of the latest scheduled scan all process
|
||||
tags:
|
||||
- scanAll
|
||||
operationId: getLatestScheduledScanAllMetrics
|
||||
deprecated: true
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
schema:
|
||||
$ref: '#/definitions/Stats'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'412':
|
||||
$ref: '#/responses/412'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
/systeminfo:
|
||||
get:
|
||||
summary: Get general system info
|
||||
|
|
@ -2181,6 +2222,80 @@ paths:
|
|||
$ref: '#/responses/403'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
/system/scanAll/schedule:
|
||||
get:
|
||||
summary: Get scan all's schedule.
|
||||
description: This endpoint is for getting a schedule for the scan all job, which scans all of images in Harbor.
|
||||
tags:
|
||||
- scanAll
|
||||
operationId: getScanAllSchedule
|
||||
responses:
|
||||
'200':
|
||||
description: Get a schedule for the scan all job, which scans all of images in Harbor.
|
||||
schema:
|
||||
$ref: '#/definitions/Schedule'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'412':
|
||||
$ref: '#/responses/412'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
put:
|
||||
summary: Update scan all's schedule.
|
||||
description: This endpoint is for updating the schedule of scan all job, which scans all of images in Harbor.
|
||||
parameters:
|
||||
- name: schedule
|
||||
in: body
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/Schedule'
|
||||
description: Updates the schedule of scan all job, which scans all of images in Harbor.
|
||||
tags:
|
||||
- scanAll
|
||||
operationId: updateScanAllSchedule
|
||||
responses:
|
||||
'200':
|
||||
$ref: '#/responses/200'
|
||||
'400':
|
||||
$ref: '#/responses/400'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'412':
|
||||
$ref: '#/responses/412'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
post:
|
||||
summary: Create a schedule or a manual trigger for the scan all job.
|
||||
description: This endpoint is for creating a schedule or a manual trigger for the scan all job, which scans all of images in Harbor.
|
||||
parameters:
|
||||
- name: schedule
|
||||
in: body
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/definitions/Schedule'
|
||||
description: Create a schedule or a manual trigger for the scan all job.
|
||||
tags:
|
||||
- scanAll
|
||||
operationId: createScanAllSchedule
|
||||
responses:
|
||||
'201':
|
||||
$ref: '#/responses/201'
|
||||
'400':
|
||||
$ref: '#/responses/400'
|
||||
'401':
|
||||
$ref: '#/responses/401'
|
||||
'403':
|
||||
$ref: '#/responses/403'
|
||||
'409':
|
||||
$ref: '#/responses/409'
|
||||
'412':
|
||||
$ref: '#/responses/412'
|
||||
'500':
|
||||
$ref: '#/responses/500'
|
||||
/ping:
|
||||
get:
|
||||
summary: Ping Harbor to check if it's alive.
|
||||
|
|
@ -3563,11 +3678,29 @@ definitions:
|
|||
Schedule:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
type: integer
|
||||
description: The id of the schedule.
|
||||
readOnly: true
|
||||
status:
|
||||
type: string
|
||||
description: The status of the schedule.
|
||||
readOnly: true
|
||||
creation_time:
|
||||
type: string
|
||||
format: date-time
|
||||
description: the creation time of the schedule.
|
||||
readOnly: true
|
||||
update_time:
|
||||
type: string
|
||||
format: date-time
|
||||
description: the update time of the schedule.
|
||||
readOnly: true
|
||||
schedule:
|
||||
$ref: '#/definitions/ScheduleObj'
|
||||
parameters:
|
||||
type: object
|
||||
description: The parameters of admin job
|
||||
description: The parameters of schedule job
|
||||
additionalProperties:
|
||||
type: object
|
||||
ScheduleObj:
|
||||
|
|
@ -3576,8 +3709,47 @@ definitions:
|
|||
type:
|
||||
type: string
|
||||
description: |
|
||||
The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manually' and 'None'.
|
||||
'Manually' means to trigger it right away and 'None' means to cancel the schedule.
|
||||
The schedule type. The valid values are 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manual' and 'None'.
|
||||
'Manual' means to trigger it right away and 'None' means to cancel the schedule.
|
||||
enum:
|
||||
- Hourly
|
||||
- Daily
|
||||
- Weekly
|
||||
- Custom
|
||||
- Manual
|
||||
- None
|
||||
cron:
|
||||
type: string
|
||||
description: A cron expression, a time-based job scheduler.
|
||||
|
||||
Stats:
|
||||
type: object
|
||||
description: Stats provides the overall progress of the scan all process.
|
||||
properties:
|
||||
total:
|
||||
type: integer
|
||||
format: int
|
||||
description: 'The total number of scan processes triggered by the scan all action'
|
||||
example: 100
|
||||
x-omitempty: false
|
||||
completed:
|
||||
type: integer
|
||||
format: int
|
||||
description: 'The number of the finished scan processes triggered by the scan all action'
|
||||
example: 90
|
||||
x-omitempty: false
|
||||
metrics:
|
||||
type: object
|
||||
description: 'The metrics data for the each status'
|
||||
additionalProperties:
|
||||
type: integer
|
||||
format: int
|
||||
example: 10
|
||||
example:
|
||||
'Success': 5
|
||||
'Error': 2
|
||||
'Running': 3
|
||||
ongoing:
|
||||
type: boolean
|
||||
description: A flag indicating job status of scan all.
|
||||
x-omitempty: false
|
||||
|
|
|
|||
|
|
@ -122,7 +122,6 @@ func init() {
|
|||
beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping")
|
||||
beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List")
|
||||
beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete")
|
||||
beego.Router("/api/system/scanAll/schedule", &ScanAllAPI{}, "get:Get;put:Put;post:Post")
|
||||
beego.Router("/api/system/CVEAllowlist", &SysCVEAllowlistAPI{}, "get:Get;put:Put")
|
||||
beego.Router("/api/system/oidc/ping", &OIDCAPI{}, "post:Ping")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,284 +0,0 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/goharbor/harbor/src/controller/scan"
|
||||
"github.com/goharbor/harbor/src/controller/scanner"
|
||||
"github.com/goharbor/harbor/src/core/api/models"
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/errors"
|
||||
"github.com/goharbor/harbor/src/lib/q"
|
||||
"github.com/goharbor/harbor/src/pkg/scan/all"
|
||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
)
|
||||
|
||||
// ScanAllAPI handles request of scan all images...
|
||||
type ScanAllAPI struct {
|
||||
BaseController
|
||||
}
|
||||
|
||||
// Prepare validates the URL and parms, it needs the system admin permission.
|
||||
func (sc *ScanAllAPI) Prepare() {
|
||||
sc.BaseController.Prepare()
|
||||
|
||||
if !sc.SecurityCtx.IsAuthenticated() {
|
||||
sc.SendUnAuthorizedError(errors.New("UnAuthorized"))
|
||||
return
|
||||
}
|
||||
if !sc.SecurityCtx.IsSysAdmin() {
|
||||
sc.SendForbiddenError(errors.New(sc.SecurityCtx.GetUsername()))
|
||||
return
|
||||
}
|
||||
|
||||
enabled, err := isScanEnabled()
|
||||
if err != nil {
|
||||
sc.SendInternalServerError(err)
|
||||
return
|
||||
}
|
||||
|
||||
if !enabled {
|
||||
sc.SendStatusServiceUnavailableError(errors.New("no scanner is configured, it's not possible to scan"))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Post according to the request, it creates a cron schedule or a manual trigger for scan all.
|
||||
// create a daily schedule for scan all
|
||||
// {
|
||||
// "schedule": {
|
||||
// "type": "Daily",
|
||||
// "cron": "0 0 0 * * *"
|
||||
// }
|
||||
// }
|
||||
// create a manual trigger for scan all
|
||||
// {
|
||||
// "schedule": {
|
||||
// "type": "Manual"
|
||||
// }
|
||||
// }
|
||||
func (sc *ScanAllAPI) Post() {
|
||||
ajr := models.AdminJobReq{}
|
||||
isValid, err := sc.DecodeJSONReqAndValidate(&ajr)
|
||||
if !isValid {
|
||||
sc.SendBadRequestError(err)
|
||||
return
|
||||
}
|
||||
|
||||
if ajr.Schedule == nil {
|
||||
sc.SendBadRequestError(fmt.Errorf("schedule is required"))
|
||||
return
|
||||
}
|
||||
|
||||
if ajr.Schedule.Type == models.ScheduleNone {
|
||||
return
|
||||
}
|
||||
|
||||
if ajr.IsPeriodic() {
|
||||
schedule, err := sc.getScanAllSchedule()
|
||||
if err != nil {
|
||||
sc.SendError(err)
|
||||
return
|
||||
}
|
||||
|
||||
if schedule != nil {
|
||||
err := errors.New("fail to set schedule for scan all as always had one, please delete it firstly then to re-schedule")
|
||||
sc.SendPreconditionFailedError(err)
|
||||
return
|
||||
}
|
||||
|
||||
scheduleID, err := sc.createOrUpdateScanAllSchedule(ajr.Schedule.Type, ajr.Schedule.Cron, nil)
|
||||
if err != nil {
|
||||
sc.SendError(err)
|
||||
return
|
||||
}
|
||||
|
||||
sc.Redirect(http.StatusCreated, strconv.FormatInt(scheduleID, 10))
|
||||
} else {
|
||||
execution, err := sc.getLatestScanAllExecution(task.ExecutionTriggerManual)
|
||||
if err != nil {
|
||||
sc.SendError(err)
|
||||
return
|
||||
}
|
||||
|
||||
if execution != nil && execution.IsOnGoing() {
|
||||
err := errors.Errorf("a previous scan all job aleady exits, its status is %s", execution.Status)
|
||||
sc.SendConflictError(err)
|
||||
return
|
||||
}
|
||||
|
||||
executionID, err := scan.DefaultController.ScanAll(sc.Context(), task.ExecutionTriggerManual, true)
|
||||
if err != nil {
|
||||
sc.SendError(err)
|
||||
return
|
||||
}
|
||||
|
||||
sc.Redirect(http.StatusCreated, strconv.FormatInt(executionID, 10))
|
||||
}
|
||||
}
|
||||
|
||||
// Put handles scan all cron schedule update/delete.
|
||||
// Request: delete the schedule of scan all
|
||||
// {
|
||||
// "schedule": {
|
||||
// "type": "None",
|
||||
// "cron": ""
|
||||
// }
|
||||
// }
|
||||
func (sc *ScanAllAPI) Put() {
|
||||
ajr := models.AdminJobReq{}
|
||||
isValid, err := sc.DecodeJSONReqAndValidate(&ajr)
|
||||
if !isValid {
|
||||
sc.SendBadRequestError(err)
|
||||
return
|
||||
}
|
||||
|
||||
if ajr.Schedule.Type == models.ScheduleManual {
|
||||
err := fmt.Errorf("fail to update scan all schedule as wrong schedule type: %s", ajr.Schedule.Type)
|
||||
sc.SendBadRequestError(err)
|
||||
return
|
||||
}
|
||||
|
||||
schedule, err := sc.getScanAllSchedule()
|
||||
if err != nil {
|
||||
sc.SendError(err)
|
||||
return
|
||||
}
|
||||
|
||||
if ajr.Schedule.Type == models.ScheduleNone {
|
||||
if schedule != nil {
|
||||
err = scheduler.Sched.UnScheduleByID(sc.Context(), schedule.ID)
|
||||
}
|
||||
} else {
|
||||
_, err = sc.createOrUpdateScanAllSchedule(ajr.Schedule.Type, ajr.Schedule.Cron, schedule)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
sc.SendError(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Get gets scan all schedule ...
|
||||
func (sc *ScanAllAPI) Get() {
|
||||
result := models.AdminJobRep{}
|
||||
|
||||
schedule, err := sc.getScanAllSchedule()
|
||||
if err != nil {
|
||||
sc.SendError(err)
|
||||
return
|
||||
}
|
||||
|
||||
if schedule != nil {
|
||||
result.ID = schedule.ID
|
||||
result.Status = schedule.Status
|
||||
result.CreationTime = schedule.CreationTime
|
||||
result.UpdateTime = schedule.UpdateTime
|
||||
result.Schedule = &models.ScheduleParam{
|
||||
Type: schedule.CRONType,
|
||||
Cron: schedule.CRON,
|
||||
}
|
||||
}
|
||||
|
||||
sc.Data["json"] = result
|
||||
sc.ServeJSON()
|
||||
}
|
||||
|
||||
// GetScheduleMetrics returns the progress metrics for the latest scheduled scan all job
|
||||
func (sc *ScanAllAPI) GetScheduleMetrics() {
|
||||
sc.getMetrics(task.ExecutionTriggerSchedule)
|
||||
}
|
||||
|
||||
// GetScanAllMetrics returns the progress metrics for the latest manually triggered scan all job
|
||||
func (sc *ScanAllAPI) GetScanAllMetrics() {
|
||||
sc.getMetrics(task.ExecutionTriggerManual)
|
||||
}
|
||||
|
||||
func (sc *ScanAllAPI) getMetrics(trigger string) {
|
||||
execution, err := sc.getLatestScanAllExecution(trigger)
|
||||
if err != nil {
|
||||
sc.SendError(err)
|
||||
return
|
||||
}
|
||||
|
||||
sts := &all.Stats{}
|
||||
if execution != nil && execution.Metrics != nil {
|
||||
metrics := execution.Metrics
|
||||
sts.Total = uint(metrics.TaskCount)
|
||||
sts.Completed = uint(metrics.SuccessTaskCount)
|
||||
sts.Metrics = map[string]uint{
|
||||
"Pending": uint(metrics.PendingTaskCount),
|
||||
"Running": uint(metrics.RunningTaskCount),
|
||||
"Success": uint(metrics.SuccessTaskCount),
|
||||
"Error": uint(metrics.ErrorTaskCount),
|
||||
"Stopped": uint(metrics.StoppedTaskCount),
|
||||
}
|
||||
sts.Ongoing = !job.Status(execution.Status).Final() || sts.Total != sts.Completed
|
||||
}
|
||||
|
||||
sc.Data["json"] = sts
|
||||
sc.ServeJSON()
|
||||
}
|
||||
|
||||
func (sc *ScanAllAPI) getScanAllSchedule() (*scheduler.Schedule, error) {
|
||||
query := q.New(q.KeyWords{"vendor_type": job.ImageScanAllJob})
|
||||
schedules, err := scheduler.Sched.ListSchedules(sc.Context(), query.First("-creation_time"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(schedules) > 1 {
|
||||
msg := "found more than one scheduled scan all job, please ensure that only one schedule left"
|
||||
return nil, errors.BadRequestError(nil).WithMessage(msg)
|
||||
} else if len(schedules) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return schedules[0], nil
|
||||
}
|
||||
|
||||
func (sc *ScanAllAPI) createOrUpdateScanAllSchedule(cronType, cron string, previous *scheduler.Schedule) (int64, error) {
|
||||
if previous != nil {
|
||||
if cronType == previous.CRONType && cron == previous.CRON {
|
||||
return previous.ID, nil
|
||||
}
|
||||
|
||||
if err := scheduler.Sched.UnScheduleByID(sc.Context(), previous.ID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
return scheduler.Sched.Schedule(sc.Context(), job.ImageScanAllJob, 0, cronType, cron, scan.ScanAllCallback, nil, nil)
|
||||
}
|
||||
|
||||
func (sc *ScanAllAPI) getLatestScanAllExecution(trigger string) (*task.Execution, error) {
|
||||
query := q.New(q.KeyWords{"vendor_type": job.ImageScanAllJob, "trigger": trigger})
|
||||
executions, err := task.ExecMgr.List(sc.Context(), query.First("-start_time"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(executions) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return executions[0], nil
|
||||
}
|
||||
|
||||
func isScanEnabled() (bool, error) {
|
||||
kws := make(map[string]interface{})
|
||||
kws["is_default"] = true
|
||||
|
||||
query := &q.Query{
|
||||
Keywords: kws,
|
||||
}
|
||||
|
||||
l, err := scanner.DefaultController.ListRegistrations(query)
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "scan all API: check if scan is enabled")
|
||||
}
|
||||
|
||||
return len(l) > 0, nil
|
||||
}
|
||||
|
|
@ -1,83 +0,0 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
|
||||
sc "github.com/goharbor/harbor/src/pkg/scan/scanner"
|
||||
"github.com/goharbor/harbor/src/testing/apitests/apilib"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
var adminJob002 apilib.AdminJobReq
|
||||
|
||||
// ScanAllAPITestSuite is a test suite to test scan all API.
|
||||
type ScanAllAPITestSuite struct {
|
||||
suite.Suite
|
||||
|
||||
m sc.Manager
|
||||
uuid string
|
||||
}
|
||||
|
||||
// TestScanAllAPI is an entry point for ScanAllAPITestSuite.
|
||||
func TestScanAllAPI(t *testing.T) {
|
||||
suite.Run(t, &ScanAllAPITestSuite{})
|
||||
}
|
||||
|
||||
// SetupSuite prepares env for test suite.
|
||||
func (suite *ScanAllAPITestSuite) SetupSuite() {
|
||||
// Ensure scanner is there
|
||||
reg := &scanner.Registration{
|
||||
Name: "Trivy",
|
||||
Description: "The trivy scanner adapter",
|
||||
URL: "https://trivy.com:8080",
|
||||
Disabled: false,
|
||||
IsDefault: true,
|
||||
}
|
||||
|
||||
scMgr := sc.New()
|
||||
uuid, err := scMgr.Create(reg)
|
||||
require.NoError(suite.T(), err, "failed to initialize trivy scanner")
|
||||
|
||||
suite.uuid = uuid
|
||||
suite.m = scMgr
|
||||
}
|
||||
|
||||
// TearDownSuite clears env for the test suite.
|
||||
func (suite *ScanAllAPITestSuite) TearDownSuite() {
|
||||
err := suite.m.Delete(suite.uuid)
|
||||
suite.NoError(err, "clear scanner")
|
||||
}
|
||||
|
||||
func (suite *ScanAllAPITestSuite) TestScanAllPost() {
|
||||
apiTest := newHarborAPI()
|
||||
|
||||
// case 1: add a new scan all job
|
||||
adminJob002.Schedule = &apilib.ScheduleParam{Type: "Manual"}
|
||||
code, err := apiTest.AddScanAll(*admin, adminJob002)
|
||||
require.NoError(suite.T(), err, "Error occurred while add a scan all job")
|
||||
suite.Equal(201, code, "Add scan all status should be 200")
|
||||
}
|
||||
|
||||
func (suite *ScanAllAPITestSuite) TestScanAllGet() {
|
||||
apiTest := newHarborAPI()
|
||||
|
||||
code, _, err := apiTest.ScanAllScheduleGet(*admin)
|
||||
require.NoError(suite.T(), err, "Error occurred while get a scan all job")
|
||||
suite.Equal(200, code, "Get scan all status should be 200")
|
||||
}
|
||||
|
|
@ -136,20 +136,19 @@ func (s *scheduler) schedule(ctx context.Context, vendorType string, vendorID in
|
|||
CreationTime: now,
|
||||
UpdateTime: now,
|
||||
}
|
||||
if params != nil {
|
||||
paramsData, err := json.Marshal(params)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
sched.CallbackFuncParam = string(paramsData)
|
||||
|
||||
paramsData, err := json.Marshal(params)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if extras != nil {
|
||||
extrasData, err := json.Marshal(extras)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
sched.ExtraAttrs = string(extrasData)
|
||||
sched.CallbackFuncParam = string(paramsData)
|
||||
|
||||
extrasData, err := json.Marshal(extras)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
sched.ExtraAttrs = string(extrasData)
|
||||
|
||||
// create schedule record
|
||||
// when checkin hook comes, the database record must exist,
|
||||
// so the database record must be created first before submitting job
|
||||
|
|
|
|||
|
|
@ -4,18 +4,18 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/go-openapi/runtime/middleware"
|
||||
"github.com/goharbor/harbor/src/controller/gc"
|
||||
"github.com/goharbor/harbor/src/core/config"
|
||||
"github.com/goharbor/harbor/src/lib/errors"
|
||||
"github.com/goharbor/harbor/src/lib/q"
|
||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
"github.com/goharbor/harbor/src/server/v2.0/handler/model"
|
||||
"github.com/goharbor/harbor/src/server/v2.0/models"
|
||||
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/gc"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type gcAPI struct {
|
||||
|
|
@ -116,13 +116,13 @@ func (g *gcAPI) updateSchedule(ctx context.Context, cronType, cron string, polic
|
|||
func (g *gcAPI) GetGCSchedule(ctx context.Context, params operation.GetGCScheduleParams) middleware.Responder {
|
||||
schedule, err := g.gcCtr.GetSchedule(ctx)
|
||||
if errors.IsNotFoundErr(err) {
|
||||
return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(&scheduler.Schedule{}).ToSwagger())
|
||||
return operation.NewGetGCScheduleOK()
|
||||
}
|
||||
if err != nil {
|
||||
return g.SendError(ctx, err)
|
||||
}
|
||||
|
||||
return operation.NewGetGCScheduleOK().WithPayload(model.NewSchedule(schedule).ToSwagger())
|
||||
return operation.NewGetGCScheduleOK().WithPayload(model.NewGCSchedule(schedule).ToSwagger())
|
||||
}
|
||||
|
||||
func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryParams) middleware.Responder {
|
||||
|
|
@ -171,7 +171,7 @@ func (g *gcAPI) GetGCHistory(ctx context.Context, params operation.GetGCHistoryP
|
|||
}
|
||||
|
||||
func (g *gcAPI) GetGC(ctx context.Context, params operation.GetGCParams) middleware.Responder {
|
||||
exec, err := g.gcCtr.GetExecution(ctx, params.GcID)
|
||||
exec, err := g.gcCtr.GetExecution(ctx, params.GCID)
|
||||
if err != nil {
|
||||
return g.SendError(ctx, err)
|
||||
}
|
||||
|
|
@ -199,13 +199,13 @@ func (g *gcAPI) GetGC(ctx context.Context, params operation.GetGCParams) middlew
|
|||
|
||||
func (g *gcAPI) GetGCLog(ctx context.Context, params operation.GetGCLogParams) middleware.Responder {
|
||||
tasks, err := g.gcCtr.ListTasks(ctx, q.New(q.KeyWords{
|
||||
"ExecutionID": params.GcID,
|
||||
"ExecutionID": params.GCID,
|
||||
}))
|
||||
if err != nil {
|
||||
return g.SendError(ctx, err)
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return g.SendError(ctx, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("garbage collection %d log is not found", params.GcID))
|
||||
return g.SendError(ctx, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("garbage collection %d log is not found", params.GCID))
|
||||
}
|
||||
log, err := g.gcCtr.GetTaskLog(ctx, tasks[0].ID)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ import (
|
|||
"net/http"
|
||||
|
||||
lib_http "github.com/goharbor/harbor/src/lib/http"
|
||||
|
||||
"github.com/goharbor/harbor/src/server/middleware"
|
||||
"github.com/goharbor/harbor/src/server/middleware/blob"
|
||||
"github.com/goharbor/harbor/src/server/middleware/quota"
|
||||
|
|
@ -33,6 +32,7 @@ func New() http.Handler {
|
|||
RepositoryAPI: newRepositoryAPI(),
|
||||
AuditlogAPI: newAuditLogAPI(),
|
||||
ScanAPI: newScanAPI(),
|
||||
ScanAllAPI: newScanAllAPI(),
|
||||
ProjectAPI: newProjectAPI(),
|
||||
PreheatAPI: newPreheatAPI(),
|
||||
IconAPI: newIconAPI(),
|
||||
|
|
@ -41,7 +41,7 @@ func New() http.Handler {
|
|||
ReplicationAPI: newReplicationAPI(),
|
||||
SysteminfoAPI: newSystemInfoAPI(),
|
||||
PingAPI: newPingAPI(),
|
||||
GcAPI: newGCAPI(),
|
||||
GCAPI: newGCAPI(),
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
|
|
|
|||
|
|
@ -2,11 +2,13 @@ package model
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-openapi/strfmt"
|
||||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||
"github.com/goharbor/harbor/src/server/v2.0/models"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ScheduleParam defines the parameter of schedule trigger
|
||||
|
|
@ -41,44 +43,49 @@ func (h *GCHistory) ToSwagger() *models.GCHistory {
|
|||
Deleted: h.Deleted,
|
||||
JobStatus: h.Status,
|
||||
Schedule: &models.ScheduleObj{
|
||||
// covert MANUAL to Manual because the type of the ScheduleObj
|
||||
// must be 'Hourly', 'Daily', 'Weekly', 'Custom', 'Manual' and 'None'
|
||||
Type: strings.Title(strings.ToLower(h.Schedule.Type)),
|
||||
Cron: h.Schedule.Cron,
|
||||
Type: h.Schedule.Type,
|
||||
},
|
||||
CreationTime: strfmt.DateTime(h.CreationTime),
|
||||
UpdateTime: strfmt.DateTime(h.UpdateTime),
|
||||
}
|
||||
}
|
||||
|
||||
// Schedule ...
|
||||
type Schedule struct {
|
||||
// GCSchedule ...
|
||||
type GCSchedule struct {
|
||||
*scheduler.Schedule
|
||||
}
|
||||
|
||||
// ToSwagger converts the schedule to the swagger model
|
||||
// TODO remove the hard code when after issue https://github.com/goharbor/harbor/issues/13047 is resolved.
|
||||
func (s *Schedule) ToSwagger() *models.GCHistory {
|
||||
func (s *GCSchedule) ToSwagger() *models.GCHistory {
|
||||
if s.Schedule == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
e, err := json.Marshal(s.ExtraAttrs)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
|
||||
return &models.GCHistory{
|
||||
ID: 0,
|
||||
ID: s.ID,
|
||||
JobName: "",
|
||||
JobKind: s.CRON,
|
||||
JobParameters: string(e),
|
||||
Deleted: false,
|
||||
JobStatus: "",
|
||||
JobStatus: s.Status,
|
||||
Schedule: &models.ScheduleObj{
|
||||
Cron: s.CRON,
|
||||
Type: "Custom",
|
||||
Type: s.CRONType,
|
||||
},
|
||||
CreationTime: strfmt.DateTime(s.CreationTime),
|
||||
UpdateTime: strfmt.DateTime(s.UpdateTime),
|
||||
}
|
||||
}
|
||||
|
||||
// NewSchedule ...
|
||||
func NewSchedule(s *scheduler.Schedule) *Schedule {
|
||||
return &Schedule{Schedule: s}
|
||||
// NewGCSchedule ...
|
||||
func NewGCSchedule(s *scheduler.Schedule) *GCSchedule {
|
||||
return &GCSchedule{Schedule: s}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"github.com/go-openapi/strfmt"
|
||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||
"github.com/goharbor/harbor/src/server/v2.0/models"
|
||||
)
|
||||
|
||||
// Schedule model
|
||||
type Schedule struct {
|
||||
*scheduler.Schedule
|
||||
}
|
||||
|
||||
// ToSwagger converts the schedule to the swagger model
|
||||
func (s *Schedule) ToSwagger() *models.Schedule {
|
||||
if s.Schedule == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &models.Schedule{
|
||||
ID: s.ID,
|
||||
Status: s.Status,
|
||||
Schedule: &models.ScheduleObj{
|
||||
Cron: s.CRON,
|
||||
Type: s.CRONType,
|
||||
},
|
||||
Parameters: s.ExtraAttrs,
|
||||
CreationTime: strfmt.DateTime(s.CreationTime),
|
||||
UpdateTime: strfmt.DateTime(s.UpdateTime),
|
||||
}
|
||||
}
|
||||
|
||||
// NewSchedule new schedule instance
|
||||
func NewSchedule(schedule *scheduler.Schedule) *Schedule {
|
||||
return &Schedule{Schedule: schedule}
|
||||
}
|
||||
|
|
@ -0,0 +1,248 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/go-openapi/runtime/middleware"
|
||||
"github.com/goharbor/harbor/src/controller/scan"
|
||||
"github.com/goharbor/harbor/src/controller/scanner"
|
||||
"github.com/goharbor/harbor/src/jobservice/job"
|
||||
"github.com/goharbor/harbor/src/lib/errors"
|
||||
"github.com/goharbor/harbor/src/lib/q"
|
||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
"github.com/goharbor/harbor/src/server/v2.0/handler/model"
|
||||
"github.com/goharbor/harbor/src/server/v2.0/models"
|
||||
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/scan_all"
|
||||
)
|
||||
|
||||
func newScanAllAPI() *scanAllAPI {
|
||||
return &scanAllAPI{
|
||||
execMgr: task.ExecMgr,
|
||||
scanCtl: scan.DefaultController,
|
||||
scannerCtl: scanner.DefaultController,
|
||||
scheduler: scheduler.Sched,
|
||||
}
|
||||
}
|
||||
|
||||
type scanAllAPI struct {
|
||||
BaseAPI
|
||||
execMgr task.ExecutionManager
|
||||
scanCtl scan.Controller
|
||||
scannerCtl scanner.Controller
|
||||
scheduler scheduler.Scheduler
|
||||
}
|
||||
|
||||
func (s *scanAllAPI) Prepare(ctx context.Context, operation string, params interface{}) middleware.Responder {
|
||||
if err := s.RequireSysAdmin(ctx); err != nil {
|
||||
return s.SendError(ctx, err)
|
||||
}
|
||||
|
||||
if err := s.requireScanEnabled(ctx); err != nil {
|
||||
return s.SendError(ctx, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scanAllAPI) CreateScanAllSchedule(ctx context.Context, params operation.CreateScanAllScheduleParams) middleware.Responder {
|
||||
req := params.Schedule
|
||||
|
||||
if req.Schedule.Type == ScheduleNone {
|
||||
return operation.NewCreateScanAllScheduleCreated()
|
||||
}
|
||||
|
||||
if req.Schedule.Type == ScheduleManual {
|
||||
execution, err := s.getLatestScanAllExecution(ctx, task.ExecutionTriggerManual)
|
||||
if err != nil {
|
||||
return s.SendError(ctx, err)
|
||||
}
|
||||
|
||||
if execution != nil && execution.IsOnGoing() {
|
||||
message := fmt.Sprintf("a previous scan all job aleady exits, its status is %s", execution.Status)
|
||||
return s.SendError(ctx, errors.ConflictError(nil).WithMessage(message))
|
||||
}
|
||||
|
||||
if _, err := s.scanCtl.ScanAll(ctx, task.ExecutionTriggerManual, true); err != nil {
|
||||
return s.SendError(ctx, err)
|
||||
}
|
||||
} else {
|
||||
schedule, err := s.getScanAllSchedule(ctx)
|
||||
if err != nil {
|
||||
return s.SendError(ctx, err)
|
||||
}
|
||||
|
||||
if schedule != nil {
|
||||
message := "fail to set schedule for scan all as always had one, please delete it firstly then to re-schedule"
|
||||
return s.SendError(ctx, errors.PreconditionFailedError(nil).WithMessage(message))
|
||||
}
|
||||
|
||||
if _, err := s.createOrUpdateScanAllSchedule(ctx, req.Schedule.Type, req.Schedule.Cron, nil); err != nil {
|
||||
return s.SendError(ctx, err)
|
||||
}
|
||||
}
|
||||
|
||||
return operation.NewCreateScanAllScheduleCreated()
|
||||
}
|
||||
|
||||
func (s *scanAllAPI) UpdateScanAllSchedule(ctx context.Context, params operation.UpdateScanAllScheduleParams) middleware.Responder {
|
||||
req := params.Schedule
|
||||
|
||||
if req.Schedule.Type == ScheduleManual {
|
||||
message := fmt.Sprintf("fail to update scan all schedule as wrong schedule type: %s", req.Schedule.Type)
|
||||
return s.SendError(ctx, errors.BadRequestError(nil).WithMessage(message))
|
||||
}
|
||||
|
||||
schedule, err := s.getScanAllSchedule(ctx)
|
||||
if err != nil {
|
||||
return s.SendError(ctx, err)
|
||||
}
|
||||
|
||||
if req.Schedule.Type == ScheduleNone {
|
||||
if schedule != nil {
|
||||
err = s.scheduler.UnScheduleByID(ctx, schedule.ID)
|
||||
}
|
||||
} else {
|
||||
_, err = s.createOrUpdateScanAllSchedule(ctx, req.Schedule.Type, req.Schedule.Cron, schedule)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return s.SendError(ctx, err)
|
||||
}
|
||||
|
||||
return operation.NewUpdateScanAllScheduleOK()
|
||||
}
|
||||
|
||||
func (s *scanAllAPI) GetScanAllSchedule(ctx context.Context, params operation.GetScanAllScheduleParams) middleware.Responder {
|
||||
schedule, err := s.getScanAllSchedule(ctx)
|
||||
if err != nil {
|
||||
return s.SendError(ctx, err)
|
||||
}
|
||||
|
||||
return operation.NewGetScanAllScheduleOK().WithPayload(model.NewSchedule(schedule).ToSwagger())
|
||||
}
|
||||
|
||||
func (s *scanAllAPI) GetLatestScanAllMetrics(ctx context.Context, params operation.GetLatestScanAllMetricsParams) middleware.Responder {
|
||||
stats, err := s.getMetrics(ctx)
|
||||
if err != nil {
|
||||
return s.SendError(ctx, err)
|
||||
}
|
||||
|
||||
return operation.NewGetLatestScanAllMetricsOK().WithPayload(stats)
|
||||
}
|
||||
|
||||
func (s *scanAllAPI) GetLatestScheduledScanAllMetrics(ctx context.Context, params operation.GetLatestScheduledScanAllMetricsParams) middleware.Responder {
|
||||
stats, err := s.getMetrics(ctx, task.ExecutionTriggerSchedule)
|
||||
if err != nil {
|
||||
return s.SendError(ctx, err)
|
||||
}
|
||||
|
||||
return operation.NewGetLatestScanAllMetricsOK().WithPayload(stats)
|
||||
}
|
||||
|
||||
func (s *scanAllAPI) createOrUpdateScanAllSchedule(ctx context.Context, cronType, cron string, previous *scheduler.Schedule) (int64, error) {
|
||||
if previous != nil {
|
||||
if cronType == previous.CRONType && cron == previous.CRON {
|
||||
return previous.ID, nil
|
||||
}
|
||||
|
||||
if err := s.scheduler.UnScheduleByID(ctx, previous.ID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
return s.scheduler.Schedule(ctx, job.ImageScanAllJob, 0, cronType, cron, scan.ScanAllCallback, nil, nil)
|
||||
}
|
||||
|
||||
func (s *scanAllAPI) getScanAllSchedule(ctx context.Context) (*scheduler.Schedule, error) {
|
||||
query := q.New(q.KeyWords{"vendor_type": job.ImageScanAllJob})
|
||||
schedules, err := s.scheduler.ListSchedules(ctx, query.First("-creation_time"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(schedules) > 1 {
|
||||
return nil, fmt.Errorf("found more than one scheduled scan all job, please ensure that only one schedule left")
|
||||
} else if len(schedules) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return schedules[0], nil
|
||||
}
|
||||
|
||||
func (s *scanAllAPI) getMetrics(ctx context.Context, trigger ...string) (*models.Stats, error) {
|
||||
execution, err := s.getLatestScanAllExecution(ctx, trigger...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sts := &models.Stats{}
|
||||
if execution != nil && execution.Metrics != nil {
|
||||
metrics := execution.Metrics
|
||||
sts.Total = metrics.TaskCount
|
||||
sts.Completed = metrics.SuccessTaskCount
|
||||
sts.Metrics = map[string]int64{
|
||||
"Pending": metrics.PendingTaskCount,
|
||||
"Running": metrics.RunningTaskCount,
|
||||
"Success": metrics.SuccessTaskCount,
|
||||
"Error": metrics.ErrorTaskCount,
|
||||
"Stopped": metrics.StoppedTaskCount,
|
||||
}
|
||||
sts.Ongoing = !job.Status(execution.Status).Final() || sts.Total != sts.Completed
|
||||
}
|
||||
|
||||
return sts, nil
|
||||
}
|
||||
|
||||
func (s *scanAllAPI) getLatestScanAllExecution(ctx context.Context, trigger ...string) (*task.Execution, error) {
|
||||
query := q.New(q.KeyWords{"vendor_type": job.ImageScanAllJob})
|
||||
if len(trigger) > 0 {
|
||||
query.Keywords["trigger"] = trigger[0]
|
||||
}
|
||||
|
||||
executions, err := s.execMgr.List(ctx, query.First("-start_time"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(executions) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return executions[0], nil
|
||||
}
|
||||
|
||||
func (s *scanAllAPI) requireScanEnabled(ctx context.Context) error {
|
||||
kws := make(map[string]interface{})
|
||||
kws["is_default"] = true
|
||||
|
||||
query := &q.Query{
|
||||
Keywords: kws,
|
||||
}
|
||||
|
||||
l, err := s.scannerCtl.ListRegistrations(query)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "check if scan is enabled")
|
||||
}
|
||||
|
||||
if len(l) == 0 {
|
||||
return errors.PreconditionFailedError(nil).WithMessage("no scanner is configured, it's not possible to scan")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,520 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
|
||||
"github.com/goharbor/harbor/src/pkg/scheduler"
|
||||
"github.com/goharbor/harbor/src/pkg/task"
|
||||
taskdao "github.com/goharbor/harbor/src/pkg/task/dao"
|
||||
"github.com/goharbor/harbor/src/server/v2.0/models"
|
||||
"github.com/goharbor/harbor/src/server/v2.0/restapi"
|
||||
scantesting "github.com/goharbor/harbor/src/testing/controller/scan"
|
||||
scannertesting "github.com/goharbor/harbor/src/testing/controller/scanner"
|
||||
"github.com/goharbor/harbor/src/testing/mock"
|
||||
schedulertesting "github.com/goharbor/harbor/src/testing/pkg/scheduler"
|
||||
tasktesting "github.com/goharbor/harbor/src/testing/pkg/task"
|
||||
htesting "github.com/goharbor/harbor/src/testing/server/v2.0/handler"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type ScanAllTestSuite struct {
|
||||
htesting.Suite
|
||||
|
||||
execMgr *tasktesting.ExecutionManager
|
||||
scanCtl *scantesting.Controller
|
||||
scannerCtl *scannertesting.Controller
|
||||
scheduler *schedulertesting.Scheduler
|
||||
|
||||
execution *task.Execution
|
||||
schedule *scheduler.Schedule
|
||||
}
|
||||
|
||||
func (suite *ScanAllTestSuite) SetupSuite() {
|
||||
suite.execution = &task.Execution{
|
||||
Status: "Running",
|
||||
Metrics: &taskdao.Metrics{
|
||||
TaskCount: 10,
|
||||
SuccessTaskCount: 5,
|
||||
ErrorTaskCount: 0,
|
||||
PendingTaskCount: 4,
|
||||
RunningTaskCount: 1,
|
||||
ScheduledTaskCount: 0,
|
||||
StoppedTaskCount: 0,
|
||||
},
|
||||
}
|
||||
|
||||
suite.schedule = &scheduler.Schedule{
|
||||
ID: 1,
|
||||
VendorType: "vendor_type",
|
||||
CRONType: "Daily",
|
||||
CRON: "0 0 0 * * *",
|
||||
Status: "Running",
|
||||
CreationTime: time.Now(),
|
||||
UpdateTime: time.Now(),
|
||||
}
|
||||
|
||||
suite.execMgr = &tasktesting.ExecutionManager{}
|
||||
suite.scanCtl = &scantesting.Controller{}
|
||||
suite.scannerCtl = &scannertesting.Controller{}
|
||||
suite.scheduler = &schedulertesting.Scheduler{}
|
||||
|
||||
suite.Config = &restapi.Config{
|
||||
ScanAllAPI: &scanAllAPI{
|
||||
execMgr: suite.execMgr,
|
||||
scanCtl: suite.scanCtl,
|
||||
scannerCtl: suite.scannerCtl,
|
||||
scheduler: suite.scheduler,
|
||||
},
|
||||
}
|
||||
|
||||
suite.Suite.SetupSuite()
|
||||
}
|
||||
|
||||
func (suite *ScanAllTestSuite) TestAuthorization() {
|
||||
newBody := func(body interface{}) io.Reader {
|
||||
if body == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
buf, err := json.Marshal(body)
|
||||
suite.Require().NoError(err)
|
||||
return bytes.NewBuffer(buf)
|
||||
}
|
||||
|
||||
schedule := models.Schedule{
|
||||
Schedule: &models.ScheduleObj{Type: "Manual"},
|
||||
}
|
||||
|
||||
reqs := []struct {
|
||||
method string
|
||||
url string
|
||||
body interface{}
|
||||
}{
|
||||
{http.MethodGet, "/scans/all/metrics", nil},
|
||||
{http.MethodGet, "/scans/schedule/metrics", nil},
|
||||
{http.MethodGet, "/system/scanAll/schedule", nil},
|
||||
{http.MethodPut, "/system/scanAll/schedule", schedule},
|
||||
{http.MethodPost, "/system/scanAll/schedule", schedule},
|
||||
}
|
||||
for _, req := range reqs {
|
||||
{
|
||||
// authorized required
|
||||
suite.Security.On("IsAuthenticated").Return(false).Once()
|
||||
|
||||
res, err := suite.DoReq(req.method, req.url, newBody(req.body))
|
||||
suite.NoError(err)
|
||||
suite.Equal(401, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// system admin required
|
||||
suite.Security.On("IsAuthenticated").Return(true).Once()
|
||||
suite.Security.On("IsSysAdmin").Return(false).Once()
|
||||
suite.Security.On("GetUsername").Return("username").Once()
|
||||
|
||||
res, err := suite.DoReq(req.method, req.url, newBody(req.body))
|
||||
suite.NoError(err)
|
||||
suite.Equal(403, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// default scanner required
|
||||
suite.Security.On("IsAuthenticated").Return(true).Once()
|
||||
suite.Security.On("IsSysAdmin").Return(true).Once()
|
||||
|
||||
mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return(nil, nil).Once()
|
||||
|
||||
res, err := suite.DoReq(req.method, req.url, newBody(req.body))
|
||||
suite.NoError(err)
|
||||
suite.Equal(412, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// default scanner required failed
|
||||
suite.Security.On("IsAuthenticated").Return(true).Once()
|
||||
suite.Security.On("IsSysAdmin").Return(true).Once()
|
||||
|
||||
mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return(nil, fmt.Errorf("failed")).Once()
|
||||
|
||||
res, err := suite.DoReq(req.method, req.url, newBody(req.body))
|
||||
suite.NoError(err)
|
||||
suite.Equal(500, res.StatusCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *ScanAllTestSuite) TestGetLatestScanAllMetrics() {
|
||||
times := 3
|
||||
suite.Security.On("IsAuthenticated").Return(true).Times(times)
|
||||
suite.Security.On("IsSysAdmin").Return(true).Times(times)
|
||||
mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times)
|
||||
|
||||
{
|
||||
// get scan all execution failed
|
||||
mock.OnAnything(suite.execMgr, "List").Return(nil, fmt.Errorf("failed to list executions")).Once()
|
||||
|
||||
res, err := suite.Get("/scans/all/metrics")
|
||||
suite.NoError(err)
|
||||
suite.Equal(500, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// scan all execution not found
|
||||
mock.OnAnything(suite.execMgr, "List").Return(nil, nil).Once()
|
||||
|
||||
var stats map[string]interface{}
|
||||
res, err := suite.GetJSON("/scans/all/metrics", &stats)
|
||||
suite.NoError(err)
|
||||
suite.Equal(200, res.StatusCode)
|
||||
suite.Contains(stats, "ongoing")
|
||||
}
|
||||
|
||||
{
|
||||
// scan all execution found
|
||||
mock.OnAnything(suite.execMgr, "List").Return([]*task.Execution{suite.execution}, nil).Once()
|
||||
|
||||
var stats models.Stats
|
||||
res, err := suite.GetJSON("/scans/all/metrics", &stats)
|
||||
suite.NoError(err)
|
||||
suite.Equal(200, res.StatusCode)
|
||||
suite.True(stats.Ongoing)
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *ScanAllTestSuite) TestGetLatestScheduledScanAllMetrics() {
|
||||
times := 3
|
||||
suite.Security.On("IsAuthenticated").Return(true).Times(times)
|
||||
suite.Security.On("IsSysAdmin").Return(true).Times(times)
|
||||
mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times)
|
||||
|
||||
{
|
||||
// get scan all execution failed
|
||||
mock.OnAnything(suite.execMgr, "List").Return(nil, fmt.Errorf("failed to list executions")).Once()
|
||||
|
||||
res, err := suite.Get("/scans/schedule/metrics")
|
||||
suite.NoError(err)
|
||||
suite.Equal(500, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// scan all execution not found
|
||||
mock.OnAnything(suite.execMgr, "List").Return(nil, nil).Once()
|
||||
|
||||
var stats map[string]interface{}
|
||||
res, err := suite.GetJSON("/scans/schedule/metrics", &stats)
|
||||
suite.NoError(err)
|
||||
suite.Equal(200, res.StatusCode)
|
||||
suite.Contains(stats, "ongoing")
|
||||
}
|
||||
|
||||
{
|
||||
// scan all execution found
|
||||
mock.OnAnything(suite.execMgr, "List").Return([]*task.Execution{suite.execution}, nil).Once()
|
||||
|
||||
var stats models.Stats
|
||||
res, err := suite.GetJSON("/scans/schedule/metrics", &stats)
|
||||
suite.NoError(err)
|
||||
suite.Equal(200, res.StatusCode)
|
||||
suite.True(stats.Ongoing)
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *ScanAllTestSuite) TestCreateScanAllSchedule() {
|
||||
times := 11
|
||||
suite.Security.On("IsAuthenticated").Return(true).Times(times)
|
||||
suite.Security.On("IsSysAdmin").Return(true).Times(times)
|
||||
mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times)
|
||||
|
||||
{
|
||||
// create scan all schedule no body
|
||||
res, err := suite.Post("/system/scanAll/schedule", nil)
|
||||
suite.NoError(err)
|
||||
suite.Equal(422, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// create scan all schedule with bad body
|
||||
res, err := suite.Post("/system/scanAll/schedule", bytes.NewBuffer([]byte("bad body")))
|
||||
suite.NoError(err)
|
||||
suite.Equal(422, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// create scan all schedule with ScheduleNone
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}}
|
||||
res, err := suite.PostJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(201, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// create scan all schedule with ScheduleManual but get latest scan all execution failed
|
||||
mock.OnAnything(suite.execMgr, "List").Return(nil, fmt.Errorf("list executions failed")).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}}
|
||||
res, err := suite.PostJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(500, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// create scan all schedule with ScheduleManual but a previous scan all job aleady exits
|
||||
mock.OnAnything(suite.execMgr, "List").Return([]*task.Execution{suite.execution}, nil).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}}
|
||||
res, err := suite.PostJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(409, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// create scan all schedule with ScheduleManual no previous scan all job exits
|
||||
mock.OnAnything(suite.execMgr, "List").Return(nil, nil).Once()
|
||||
mock.OnAnything(suite.scanCtl, "ScanAll").Return(int64(1), nil).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}}
|
||||
res, err := suite.PostJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(201, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// create scan all schedule with ScheduleManual but scan all failed
|
||||
mock.OnAnything(suite.execMgr, "List").Return(nil, nil).Once()
|
||||
mock.OnAnything(suite.scanCtl, "ScanAll").Return(int64(0), fmt.Errorf("scan all failed")).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}}
|
||||
res, err := suite.PostJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(500, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// create scan all schedule with periodic but get latest schedule failed
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, fmt.Errorf("get schedule failed")).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}}
|
||||
res, err := suite.PostJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(500, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// create scan all schedule with periodic but schedule areadly exists
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}}
|
||||
res, err := suite.PostJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(412, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// create scan all schedule with periodic but create schedule failed
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, nil).Once()
|
||||
mock.OnAnything(suite.scheduler, "Schedule").Return(int64(0), fmt.Errorf("create schedule failed")).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}}
|
||||
res, err := suite.PostJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(500, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// create scan all schedule with periodic
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, nil).Once()
|
||||
mock.OnAnything(suite.scheduler, "Schedule").Return(int64(1), nil).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}}
|
||||
res, err := suite.PostJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(201, res.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *ScanAllTestSuite) TestUpdateScanAllSchedule() {
|
||||
times := 11
|
||||
suite.Security.On("IsAuthenticated").Return(true).Times(times)
|
||||
suite.Security.On("IsSysAdmin").Return(true).Times(times)
|
||||
mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times)
|
||||
|
||||
{
|
||||
// update scan all schedule no body
|
||||
res, err := suite.Put("/system/scanAll/schedule", nil)
|
||||
suite.NoError(err)
|
||||
suite.Equal(422, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// update scan all schedule with bad body
|
||||
res, err := suite.Put("/system/scanAll/schedule", bytes.NewBuffer([]byte("bad body")))
|
||||
suite.NoError(err)
|
||||
suite.Equal(422, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// update scan all schedule with ScheduleManual
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleManual}}
|
||||
res, err := suite.PutJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(400, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// update scan all schedule but get schedule failed
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, fmt.Errorf("get schedule failed")).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}}
|
||||
res, err := suite.PutJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(500, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// update scan all schedule with ScheduleNone when no schedule found
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, nil).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}}
|
||||
res, err := suite.PutJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(200, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// update scan all schedule with ScheduleNone and unschedule failed
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
|
||||
mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(fmt.Errorf("unschedule failed")).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}}
|
||||
res, err := suite.PutJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(500, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// update scan all schedule with ScheduleNone successfully
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
|
||||
mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(nil).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleNone}}
|
||||
res, err := suite.PutJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(200, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// update scan all schedule with periodic but schedule not changed
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleDaily, Cron: "0 0 0 * * *"}}
|
||||
res, err := suite.PutJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(200, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// update scan all schedule with periodic and schedule changed
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
|
||||
mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(nil).Once()
|
||||
mock.OnAnything(suite.scheduler, "Schedule").Return(int64(1), nil).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleCustom, Cron: "0 1 0 * * *"}}
|
||||
res, err := suite.PutJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(200, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// update scan all schedule with periodic and schedule changed, but unschedule old schedule failed
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
|
||||
mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(fmt.Errorf("unschedule failed")).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleCustom, Cron: "0 1 0 * * *"}}
|
||||
res, err := suite.PutJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(500, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// update scan all schedule with periodic and schedule changed, but creat new schedule failed
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
|
||||
mock.OnAnything(suite.scheduler, "UnScheduleByID").Return(nil).Once()
|
||||
mock.OnAnything(suite.scheduler, "Schedule").Return(int64(0), fmt.Errorf("create schedule failed")).Once()
|
||||
|
||||
body := models.Schedule{Schedule: &models.ScheduleObj{Type: ScheduleCustom, Cron: "0 1 0 * * *"}}
|
||||
res, err := suite.PutJSON("/system/scanAll/schedule", body)
|
||||
suite.NoError(err)
|
||||
suite.Equal(500, res.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *ScanAllTestSuite) TestGetScanAllSchedule() {
|
||||
times := 4
|
||||
suite.Security.On("IsAuthenticated").Return(true).Times(times)
|
||||
suite.Security.On("IsSysAdmin").Return(true).Times(times)
|
||||
mock.OnAnything(suite.scannerCtl, "ListRegistrations").Return([]*scanner.Registration{{ID: int64(1)}}, nil).Times(times)
|
||||
|
||||
{
|
||||
// get schedule failed
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, fmt.Errorf("get schedule failed")).Once()
|
||||
|
||||
res, err := suite.Get("/system/scanAll/schedule")
|
||||
suite.NoError(err)
|
||||
suite.Equal(500, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// schedule not found
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return(nil, nil).Once()
|
||||
|
||||
res, err := suite.Get("/system/scanAll/schedule")
|
||||
suite.NoError(err)
|
||||
suite.Equal(200, res.StatusCode)
|
||||
}
|
||||
|
||||
{
|
||||
// schedule found
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule}, nil).Once()
|
||||
|
||||
var schedule models.Schedule
|
||||
res, err := suite.GetJSON("/system/scanAll/schedule", &schedule)
|
||||
suite.NoError(err)
|
||||
suite.Equal(200, res.StatusCode)
|
||||
suite.Equal(suite.schedule.CRONType, schedule.Schedule.Type)
|
||||
suite.Equal(suite.schedule.CRON, schedule.Schedule.Cron)
|
||||
}
|
||||
|
||||
{
|
||||
// schedule found more than one
|
||||
mock.OnAnything(suite.scheduler, "ListSchedules").Return([]*scheduler.Schedule{suite.schedule, suite.schedule}, nil).Once()
|
||||
|
||||
res, err := suite.Get("/system/scanAll/schedule")
|
||||
suite.NoError(err)
|
||||
suite.Equal(500, res.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanAllTestSuite(t *testing.T) {
|
||||
suite.Run(t, &ScanAllTestSuite{})
|
||||
}
|
||||
|
|
@ -45,7 +45,6 @@ func registerLegacyRoutes() {
|
|||
beego.Router("/api/"+version+"/quotas", &api.QuotaAPI{}, "get:List")
|
||||
beego.Router("/api/"+version+"/quotas/:id([0-9]+)", &api.QuotaAPI{}, "get:Get;put:Put")
|
||||
|
||||
beego.Router("/api/"+version+"/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post")
|
||||
beego.Router("/api/"+version+"/system/CVEAllowlist", &api.SysCVEAllowlistAPI{}, "get:Get;put:Put")
|
||||
beego.Router("/api/"+version+"/system/oidc/ping", &api.OIDCAPI{}, "post:Ping")
|
||||
|
||||
|
|
@ -106,9 +105,4 @@ func registerLegacyRoutes() {
|
|||
proScannerAPI := &api.ProjectScannerAPI{}
|
||||
beego.Router("/api/"+version+"/projects/:pid([0-9]+)/scanner", proScannerAPI, "get:GetProjectScanner;put:SetProjectScanner")
|
||||
beego.Router("/api/"+version+"/projects/:pid([0-9]+)/scanner/candidates", proScannerAPI, "get:GetProScannerCandidates")
|
||||
|
||||
// Add routes for scan all metrics
|
||||
scanAllAPI := &api.ScanAllAPI{}
|
||||
beego.Router("/api/"+version+"/scans/all/metrics", scanAllAPI, "get:GetScanAllMetrics")
|
||||
beego.Router("/api/"+version+"/scans/schedule/metrics", scanAllAPI, "get:GetScheduleMetrics")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,158 @@
|
|||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/security"
|
||||
lib "github.com/goharbor/harbor/src/lib/http"
|
||||
"github.com/goharbor/harbor/src/server/middleware"
|
||||
"github.com/goharbor/harbor/src/server/v2.0/restapi"
|
||||
securitytesting "github.com/goharbor/harbor/src/testing/common/security"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
// Suite ...
|
||||
type Suite struct {
|
||||
suite.Suite
|
||||
|
||||
Config *restapi.Config
|
||||
Security *securitytesting.Context
|
||||
ts *httptest.Server
|
||||
tc *http.Client
|
||||
}
|
||||
|
||||
// SetupSuite ...
|
||||
func (suite *Suite) SetupSuite() {
|
||||
h, api, _ := restapi.HandlerAPI(*suite.Config)
|
||||
|
||||
api.ServeError = func(rw http.ResponseWriter, r *http.Request, err error) {
|
||||
lib.SendError(rw, err)
|
||||
}
|
||||
|
||||
suite.Security = &securitytesting.Context{}
|
||||
m := middleware.New(func(w http.ResponseWriter, r *http.Request, next http.Handler) {
|
||||
next.ServeHTTP(w, r.WithContext(security.NewContext(r.Context(), suite.Security)))
|
||||
})
|
||||
|
||||
suite.ts = httptest.NewServer(m(h))
|
||||
suite.tc = http.DefaultClient
|
||||
}
|
||||
|
||||
// TearDownSuite ...
|
||||
func (suite *Suite) TearDownSuite() {
|
||||
suite.ts.Close()
|
||||
}
|
||||
|
||||
// DoReq ...
|
||||
func (suite *Suite) DoReq(method string, url string, body io.Reader, contentTypes ...string) (*http.Response, error) {
|
||||
req, err := http.NewRequest(method, suite.ts.URL+"/api/v2.0"+url, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
contentType := "application/json"
|
||||
if len(contentTypes) > 0 {
|
||||
contentType = contentTypes[0]
|
||||
}
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
|
||||
return suite.tc.Do(req)
|
||||
}
|
||||
|
||||
// Delete ...
|
||||
func (suite *Suite) Delete(url string, contentTypes ...string) (*http.Response, error) {
|
||||
return suite.DoReq(http.MethodDelete, url, nil, contentTypes...)
|
||||
}
|
||||
|
||||
// Get ...
|
||||
func (suite *Suite) Get(url string, contentTypes ...string) (*http.Response, error) {
|
||||
return suite.DoReq(http.MethodGet, url, nil, contentTypes...)
|
||||
}
|
||||
|
||||
// GetJSON ...
|
||||
func (suite *Suite) GetJSON(url string, js interface{}) (*http.Response, error) {
|
||||
res, err := suite.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest {
|
||||
data, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Body.Close()
|
||||
|
||||
if err := json.Unmarshal(data, js); err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
res.Body = ioutil.NopCloser(bytes.NewBuffer(data))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Patch ...
|
||||
func (suite *Suite) Patch(url string, body io.Reader, contentTypes ...string) (*http.Response, error) {
|
||||
return suite.DoReq(http.MethodPatch, url, body, contentTypes...)
|
||||
}
|
||||
|
||||
// PatchJSON ...
|
||||
func (suite *Suite) PatchJSON(url string, js interface{}) (*http.Response, error) {
|
||||
buf, err := json.Marshal(js)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return suite.Patch(url, bytes.NewBuffer(buf))
|
||||
}
|
||||
|
||||
// Post ...
|
||||
func (suite *Suite) Post(url string, body io.Reader, contentTypes ...string) (*http.Response, error) {
|
||||
return suite.DoReq(http.MethodPost, url, body, contentTypes...)
|
||||
}
|
||||
|
||||
// PostJSON ...
|
||||
func (suite *Suite) PostJSON(url string, js interface{}) (*http.Response, error) {
|
||||
buf, err := json.Marshal(js)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return suite.Post(url, bytes.NewBuffer(buf))
|
||||
}
|
||||
|
||||
// Put ...
|
||||
func (suite *Suite) Put(url string, body io.Reader, contentTypes ...string) (*http.Response, error) {
|
||||
return suite.DoReq(http.MethodPut, url, body, contentTypes...)
|
||||
}
|
||||
|
||||
// PutJSON ...
|
||||
func (suite *Suite) PutJSON(url string, js interface{}) (*http.Response, error) {
|
||||
buf, err := json.Marshal(js)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return suite.Put(url, bytes.NewBuffer(buf))
|
||||
}
|
||||
|
|
@ -28,7 +28,7 @@ def get_endpoint():
|
|||
|
||||
def _create_client(server, credential, debug, api_type="products"):
|
||||
cfg = None
|
||||
if api_type in ('projectv2', 'artifact', 'repository', 'scan', 'preheat', 'replication', 'robot', 'gc'):
|
||||
if api_type in ('projectv2', 'artifact', 'repository', 'scan', 'scanall', 'preheat', 'replication', 'robot', 'gc'):
|
||||
cfg = v2_swagger_client.Configuration()
|
||||
else:
|
||||
cfg = swagger_client.Configuration()
|
||||
|
|
@ -58,6 +58,7 @@ def _create_client(server, credential, debug, api_type="products"):
|
|||
"preheat": v2_swagger_client.PreheatApi(v2_swagger_client.ApiClient(cfg)),
|
||||
"repository": v2_swagger_client.RepositoryApi(v2_swagger_client.ApiClient(cfg)),
|
||||
"scan": v2_swagger_client.ScanApi(v2_swagger_client.ApiClient(cfg)),
|
||||
"scanall": v2_swagger_client.ScanAllApi(v2_swagger_client.ApiClient(cfg)),
|
||||
"scanner": swagger_client.ScannersApi(swagger_client.ApiClient(cfg)),
|
||||
"replication": v2_swagger_client.ReplicationApi(v2_swagger_client.ApiClient(cfg)),
|
||||
"robot": v2_swagger_client.RobotApi(v2_swagger_client.ApiClient(cfg)),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import time
|
||||
import base
|
||||
import v2_swagger_client
|
||||
from v2_swagger_client.rest import ApiException
|
||||
|
||||
class ScanAll(base.Base):
|
||||
def __init__(self):
|
||||
super(ScanAll,self).__init__(api_type="scanall")
|
||||
|
||||
def create_scan_all_schedule(self, schedule_type, cron=None, expect_status_code=201, expect_response_body=None, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
|
||||
schedule_obj = v2_swagger_client.ScheduleObj()
|
||||
schedule_obj.type = schedule_type
|
||||
if cron is not None:
|
||||
schedule_obj.cron = cron
|
||||
|
||||
schedule = v2_swagger_client.Schedule()
|
||||
schedule.schedule = schedule_obj
|
||||
|
||||
try:
|
||||
_, status_code, _ = client.create_scan_all_schedule_with_http_info(schedule)
|
||||
except ApiException as e:
|
||||
if e.status == expect_status_code:
|
||||
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
|
||||
raise Exception(r"Create scan all schedule response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
|
||||
else:
|
||||
return e.reason, e.body
|
||||
else:
|
||||
raise Exception(r"Create scan all schedule result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
|
||||
base._assert_status_code(expect_status_code, status_code)
|
||||
|
||||
def scan_all_now(self, **kwargs):
|
||||
self.create_scan_all_schedule('Manual', **kwargs)
|
||||
|
||||
def wait_until_scans_all_finish(self, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
timeout_count = 50
|
||||
while True:
|
||||
time.sleep(5)
|
||||
timeout_count = timeout_count - 1
|
||||
if (timeout_count == 0):
|
||||
break
|
||||
stats = client.get_latest_scan_all_metrics()
|
||||
print("Scan all status:", stats)
|
||||
if stats.ongoing is False:
|
||||
return
|
||||
raise Exception("Error: Scan all job is timeout.")
|
||||
|
|
@ -117,46 +117,6 @@ class System(base.Base):
|
|||
base._assert_status_code(expect_status_code, status_code)
|
||||
return base._get_id_from_header(header)
|
||||
|
||||
def wait_until_scans_all_finish(self, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
timeout_count = 50
|
||||
scan_status=""
|
||||
while True:
|
||||
time.sleep(5)
|
||||
timeout_count = timeout_count - 1
|
||||
if (timeout_count == 0):
|
||||
break
|
||||
stats = client.scans_all_metrics_get()
|
||||
print("Scan all status:", stats)
|
||||
if stats.ongoing is False:
|
||||
return
|
||||
raise Exception("Error: Scan all job is timeout.")
|
||||
|
||||
def create_scan_all_schedule(self, schedule_type, cron = None, expect_status_code = 201, expect_response_body = None, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
scanschedule = swagger_client.AdminJobScheduleObj()
|
||||
scanschedule.type = schedule_type
|
||||
if cron is not None:
|
||||
scanschedule.cron = cron
|
||||
|
||||
scan_all_schedule = swagger_client.AdminJobSchedule(scanschedule)
|
||||
try:
|
||||
_, status_code, header = client.system_scan_all_schedule_post_with_http_info(scan_all_schedule)
|
||||
except ApiException as e:
|
||||
if e.status == expect_status_code:
|
||||
if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
|
||||
raise Exception(r"Create Scan All schedule response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
|
||||
else:
|
||||
return e.reason, e.body
|
||||
else:
|
||||
raise Exception(r"Create Scan All schedule result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
|
||||
base._assert_status_code(expect_status_code, status_code)
|
||||
return base._get_id_from_header(header)
|
||||
|
||||
def scan_now(self, **kwargs):
|
||||
scan_all_id = self.create_scan_all_schedule('Manual', **kwargs)
|
||||
return scan_all_id
|
||||
|
||||
def set_cve_allowlist(self, expires_at=None, expected_status_code=200, *cve_ids, **kwargs):
|
||||
client = self._get_client(**kwargs)
|
||||
cve_list = [swagger_client.CVEAllowlistItem(cve_id=c) for c in cve_ids]
|
||||
|
|
|
|||
|
|
@ -4,20 +4,21 @@ import unittest
|
|||
from testutils import harbor_server, suppress_urllib3_warning
|
||||
from testutils import TEARDOWN
|
||||
from testutils import ADMIN_CLIENT
|
||||
from library.system import System
|
||||
from library.project import Project
|
||||
from library.user import User
|
||||
from library.repository import Repository
|
||||
from library.repository import push_self_build_image_to_project
|
||||
from library.artifact import Artifact
|
||||
from library.scan_all import ScanAll
|
||||
|
||||
class TestScanAll(unittest.TestCase):
|
||||
@suppress_urllib3_warning
|
||||
def setUp(self):
|
||||
self.system = System()
|
||||
self.project= Project()
|
||||
self.user= User()
|
||||
self.artifact = Artifact()
|
||||
self.repo = Repository()
|
||||
self.scan_all = ScanAll()
|
||||
|
||||
@unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
|
||||
def tearDown(self):
|
||||
|
|
@ -81,8 +82,8 @@ class TestScanAll(unittest.TestCase):
|
|||
TestScanAll.repo_Luca_name, tag_Luca = push_self_build_image_to_project(TestScanAll.project_Luca_name, harbor_server, user_Luca_name, user_common_password, image_b, src_tag)
|
||||
|
||||
#4. Trigger scan all event;
|
||||
self.system.scan_now(**ADMIN_CLIENT)
|
||||
self.system.wait_until_scans_all_finish(**ADMIN_CLIENT)
|
||||
self.scan_all.scan_all_now(**ADMIN_CLIENT)
|
||||
self.scan_all.wait_until_scans_all_finish(**ADMIN_CLIENT)
|
||||
|
||||
#5. Check if image in project_Alice and another image in project_Luca were both scanned.
|
||||
self.artifact.check_image_scan_result(TestScanAll.project_Alice_name, image_a, tag_Alice, **USER_ALICE_CLIENT)
|
||||
|
|
|
|||
Loading…
Reference in New Issue