Refactor scan all api (#7120)

* Refactor scan all api

This commit is to let scan all api using admin job to handle schedule
management. After the PR, GC and scan all share unified code path.

Signed-off-by: wang yan <wangyan@vmware.com>

* update admin job api code according to review comments

Signed-off-by: wang yan <wangyan@vmware.com>

* Update test code and comments per review

Signed-off-by: wang yan <wangyan@vmware.com>

docs/swagger.yaml

@@ -1405,33 +1405,6 @@ paths:
           $ref: '#/responses/UnsupportedMediaType'
         '503':
           description: Harbor is not deployed with Clair.
-  /repositories/scanAll:
-    post:
-      summary: Scan all images of the registry.
-      description: |
-        The server will launch different jobs to scan each image on the regisitry, so this is equivalent to calling  the API to scan the image one by one in background, so there's no way to track the overall status of the "scan all" action.  Only system adim has permission to call this API.
-      parameters:
-        - name: project_id
-          in: query
-          type: integer
-          description: When this parm is set only the images under the project identified by the project_id will be scanned.
-      tags:
-        - Products          
-      responses:
-        '202':
-          description: The action is successully taken in the background.  If some images are failed to scan it will only be reflected in the job status.
-        '401':
-          description: User needs to login or call the API with correct credentials.
-        '403':
-          description: User doesn't have permission to perform the action.
-        '409':
-          description: There is a "scanall" job in progress, so the request cannot be served.
-        '415':
-          $ref: '#/responses/UnsupportedMediaType'
-        '500':
-          description: Failed to initiate the action.
-        '503':
-          description: Harbor is not deployed with Clair.
   '/repositories/{repo_name}/tags/{tag}/vulnerability/details':
     get:
       summary: Get vulnerability details of the image.
@@ -2640,8 +2613,6 @@ paths:
         '200':
           description: Get gc results successfully.
           schema:
-            type: array
-            items:
               $ref: '#/definitions/GCResult'
         '401':
           description: User need to log in first.
@@ -2687,9 +2658,7 @@ paths:
         '200':
           description: Get gc's schedule.
           schema:
-            type: array
-            items:
-              $ref: '#/definitions/GCSchedule'
+              $ref: '#/definitions/AdminJobSchedule'
         '401':
           description: User need to log in first.
         '403':
@@ -2705,21 +2674,19 @@ paths:
           in: body
           required: true
           schema:
-            $ref: '#/definitions/GCSchedule'
-          description: Updates of gs's schedule.
+            $ref: '#/definitions/AdminJobSchedule'
+          description: Updates of gc's schedule.
       tags:
         - Products
       responses:
         '200':
-          description: Updated replication's target successfully.
+          description: Updated gc's schedule successfully.
         '400':
-          description: The target is associated with policy which is enabled.
+          description: Invalid schedule type.
         '401':
           description: User need to log in first.
         '403':
           description: User does not have permission of admin role.
-        '404':
-          description: Target ID does not exist.
         '500':
           description: Unexpected internal errors.
     post:
@@ -2731,23 +2698,92 @@ paths:
           in: body
           required: true
           schema:
-            $ref: '#/definitions/GCSchedule'
-          description: Updates of gs's schedule.
+            $ref: '#/definitions/AdminJobSchedule'
+          description: Updates of gc's schedule.
       tags:
         - Products
       responses:
         '200':
-          description: Updated replication's target successfully.
+          description: GC schedule successfully.
         '400':
-          description: The target is associated with policy which is enabled.
+          description: Invalid schedule type.
         '401':
           description: User need to log in first.
         '403':
           description: User does not have permission of admin role.
-        '404':
-          description: Target ID does not exist.
+        '409':
+          description: There is a "gc" job in progress, so the request cannot be served.
         '500':
           description: Unexpected internal errors.
+  /system/scanAll/schedule:
+    get:
+      summary: Get scan_all's schedule.
+      description: This endpoint is for getting a schedule for the scan all job, which scans all of images in Harbor.
+      tags:
+      - Products
+      responses:
+        '200':
+          description: Get a schedule for the scan all job, which scans all of images in Harbor.
+          schema:
+              $ref: '#/definitions/AdminJobSchedule'
+        '401':
+          description: User need to log in first.
+        '403':
+          description: Only admin has this authority.
+        '500':
+          description: Unexpected internal errors.
+    put:
+      summary: Update scan all's schedule.
+      description: |
+        This endpoint is for updating the schedule of scan all job, which scans all of images in Harbor.
+      parameters:
+      - name: schedule
+        in: body
+        required: true
+        schema:
+          $ref: '#/definitions/AdminJobSchedule'
+        description: Updates the schedule of scan all job, which scans all of images in Harbor.
+      tags:
+      - Products
+      responses:
+        '200':
+          description: Updated scan_all's schedule successfully.
+        '400':
+          description: Invalid schedule type.
+        '401':
+          description: User need to log in first.
+        '403':
+          description: User does not have permission of admin role.
+        '500':
+          description: Unexpected internal errors.
+    post:
+      summary: Create a schedule or a manual trigger for the scan all job.
+      description: |
+        This endpoint is for creating a schedule or a manual trigger for the scan all job, which scans all of images in Harbor.
+      parameters:
+      - name: schedule
+        in: body
+        required: true
+        schema:
+          $ref: '#/definitions/AdminJobSchedule'
+        description: Create a schedule or a manual trigger for the scan all job.
+      tags:
+      - Products
+      responses:
+        '200':
+          description: Updated scan_all's schedule successfully.
+        '400':
+          description: Invalid schedule type.
+        '401':
+          description: User need to log in first.
+        '403':
+          description: User does not have permission of admin role.
+        '409':
+          description: There is a "scanall" job in progress, so the request cannot be served.
+        '500':
+          description: Unexpected internal errors.
+        '503':
+          description: Harbor is not deployed with Clair.
   /configurations:
     get:
       summary: Get system configurations.
@@ -4696,7 +4732,7 @@ definitions:
         type: string
         description: the job kind of gc job.
       schedule:
-        $ref: '#/definitions/GCScheduleSchedule'
+        $ref: '#/definitions/AdminJobScheduleObj'
       job_status:
         type: string
         description: the status of gc job.
@@ -4708,13 +4744,13 @@ definitions:
         description: the creation time of gc job.
       update_time:
         type: string
-        description: the update time of gc job.        
-  GCSchedule:
+        description: the update time of gc job.
+  AdminJobSchedule:
     type: object
     properties:
       schedule:
-        $ref: '#/definitions/GCScheduleSchedule'
-  GCScheduleSchedule:
+        $ref: '#/definitions/AdminJobScheduleObj'
+  AdminJobScheduleObj:
     type: object
     properties:
       type:

src/common/utils/test/test.go

@@ -125,7 +125,7 @@ func GetUnitTestConfig() map[string]interface{} {
 		common.WithNotary:             "false",
 		common.WithChartMuseum:        "false",
 		common.SelfRegistration:       "true",
-		common.WithClair:              "false",
+		common.WithClair:              "true",
 		common.TokenServiceURL:        "http://core:8080/service/token",
 		common.RegistryURL:            fmt.Sprintf("http://%s:5000", ipAddress),
 	}

src/core/api/admin_job.go

@@ -0,0 +1,261 @@
+// Copyright 2018 Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"encoding/json"
+	"github.com/goharbor/harbor/src/common/dao"
+	common_http "github.com/goharbor/harbor/src/common/http"
+	common_job "github.com/goharbor/harbor/src/common/job"
+	common_models "github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/core/api/models"
+	utils_core "github.com/goharbor/harbor/src/core/utils"
+)
+
+// AJAPI manages the CRUD of admin job and its schedule, any API wants to handle manual and cron job like ScanAll and GC cloud reuse it.
+type AJAPI struct {
+	BaseController
+}
+
+// Prepare validates the URL and parms, it needs the system admin permission.
+func (aj *AJAPI) Prepare() {
+	aj.BaseController.Prepare()
+}
+
+// updateSchedule update a schedule of admin job.
+func (aj *AJAPI) updateSchedule(ajr models.AdminJobReq) {
+	if ajr.Schedule.Type == models.ScheduleManual {
+		aj.HandleInternalServerError(fmt.Sprintf("Fail to update admin job schedule as wrong schedule type: %s.", ajr.Schedule.Type))
+		return
+	}
+
+	query := &common_models.AdminJobQuery{
+		Name: ajr.Name,
+		Kind: common_job.JobKindPeriodic,
+	}
+	jobs, err := dao.GetAdminJobs(query)
+	if err != nil {
+		aj.HandleInternalServerError(fmt.Sprintf("%v", err))
+		return
+	}
+	if len(jobs) != 1 {
+		aj.HandleInternalServerError("Fail to update admin job schedule as we found more than one schedule in system, please ensure that only one schedule left for your job .")
+		return
+	}
+
+	// stop the scheduled job and remove it.
+	if err = utils_core.GetJobServiceClient().PostAction(jobs[0].UUID, common_job.JobActionStop); err != nil {
+		if e, ok := err.(*common_http.Error); !ok || e.Code != http.StatusNotFound {
+			aj.HandleInternalServerError(fmt.Sprintf("%v", err))
+			return
+		}
+	}
+
+	if err = dao.DeleteAdminJob(jobs[0].ID); err != nil {
+		aj.HandleInternalServerError(fmt.Sprintf("%v", err))
+		return
+	}
+
+	// Set schedule to None means to cancel the schedule, won't add new job.
+	if ajr.Schedule.Type != models.ScheduleNone {
+		aj.submit(&ajr)
+	}
+}
+
+// get get a execution of admin job by ID
+func (aj *AJAPI) get(id int64) {
+	jobs, err := dao.GetAdminJobs(&common_models.AdminJobQuery{
+		ID: id,
+	})
+	if err != nil {
+		aj.HandleInternalServerError(fmt.Sprintf("failed to get admin jobs: %v", err))
+		return
+	}
+	if len(jobs) == 0 {
+		aj.HandleNotFound("No admin job found.")
+		return
+	}
+
+	adminJobRep, err := convertToAdminJobRep(jobs[0])
+	if err != nil {
+		aj.HandleInternalServerError(fmt.Sprintf("failed to convert admin job response: %v", err))
+		return
+	}
+
+	aj.Data["json"] = adminJobRep
+	aj.ServeJSON()
+}
+
+// list list all executions of admin job by name
+func (aj *AJAPI) list(name string) {
+	jobs, err := dao.GetTop10AdminJobsOfName(name)
+	if err != nil {
+		aj.HandleInternalServerError(fmt.Sprintf("failed to get admin jobs: %v", err))
+		return
+	}
+
+	AdminJobReps := []*models.AdminJobRep{}
+	for _, job := range jobs {
+		AdminJobRep, err := convertToAdminJobRep(job)
+		if err != nil {
+			aj.HandleInternalServerError(fmt.Sprintf("failed to convert admin job response: %v", err))
+			return
+		}
+		AdminJobReps = append(AdminJobReps, &AdminJobRep)
+	}
+
+	aj.Data["json"] = AdminJobReps
+	aj.ServeJSON()
+}
+
+// getSchedule gets admin job schedule ...
+func (aj *AJAPI) getSchedule(name string) {
+	adminJobSchedule := models.AdminJobSchedule{}
+
+	jobs, err := dao.GetAdminJobs(&common_models.AdminJobQuery{
+		Name: name,
+		Kind: common_job.JobKindPeriodic,
+	})
+	if err != nil {
+		aj.HandleInternalServerError(fmt.Sprintf("failed to get admin jobs: %v", err))
+		return
+	}
+	if len(jobs) > 1 {
+		aj.HandleInternalServerError("Get more than one scheduled admin job, make sure there has only one.")
+		return
+	}
+
+	if len(jobs) != 0 {
+		adminJobRep, err := convertToAdminJobRep(jobs[0])
+		if err != nil {
+			aj.HandleInternalServerError(fmt.Sprintf("failed to convert admin job response: %v", err))
+			return
+		}
+		adminJobSchedule.Schedule = adminJobRep.Schedule
+	}
+
+	aj.Data["json"] = adminJobSchedule
+	aj.ServeJSON()
+}
+
+// getLog ...
+func (aj *AJAPI) getLog(id int64) {
+	job, err := dao.GetAdminJob(id)
+	if err != nil {
+		log.Errorf("Failed to load job data for job: %d, error: %v", id, err)
+		aj.CustomAbort(http.StatusInternalServerError, "Failed to get Job data")
+	}
+	if job == nil {
+		log.Errorf("Failed to get admin job: %d", id)
+		aj.CustomAbort(http.StatusNotFound, "Failed to get Job")
+	}
+
+	logBytes, err := utils_core.GetJobServiceClient().GetJobLog(job.UUID)
+	if err != nil {
+		if httpErr, ok := err.(*common_http.Error); ok {
+			aj.RenderError(httpErr.Code, "")
+			log.Errorf(fmt.Sprintf("failed to get log of job %d: %d %s",
+				id, httpErr.Code, httpErr.Message))
+			return
+		}
+		aj.HandleInternalServerError(fmt.Sprintf("Failed to get job logs, uuid: %s, error: %v", job.UUID, err))
+		return
+	}
+	aj.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(logBytes)))
+	aj.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain")
+	_, err = aj.Ctx.ResponseWriter.Write(logBytes)
+	if err != nil {
+		aj.HandleInternalServerError(fmt.Sprintf("Failed to write job logs, uuid: %s, error: %v", job.UUID, err))
+	}
+}
+
+// submit submits a job to job service per request
+func (aj *AJAPI) submit(ajr *models.AdminJobReq) {
+	// cannot post multiple schedule for admin job.
+	if ajr.IsPeriodic() {
+		jobs, err := dao.GetAdminJobs(&common_models.AdminJobQuery{
+			Name: ajr.Name,
+			Kind: common_job.JobKindPeriodic,
+		})
+		if err != nil {
+			aj.HandleInternalServerError(fmt.Sprintf("failed to get admin jobs: %v", err))
+			return
+		}
+		if len(jobs) != 0 {
+			aj.HandleStatusPreconditionFailed("Fail to set schedule for admin job as always had one, please delete it firstly then to re-schedule.")
+			return
+		}
+	}
+
+	id, err := dao.AddAdminJob(&common_models.AdminJob{
+		Name: ajr.Name,
+		Kind: ajr.JobKind(),
+		Cron: ajr.CronString(),
+	})
+	if err != nil {
+		aj.HandleInternalServerError(fmt.Sprintf("%v", err))
+		return
+	}
+	ajr.ID = id
+	job := ajr.ToJob()
+
+	// submit job to job service
+	log.Debugf("submitting admin job to job service")
+	uuid, err := utils_core.GetJobServiceClient().SubmitJob(job)
+	if err != nil {
+		if err := dao.DeleteAdminJob(id); err != nil {
+			log.Debugf("Failed to delete admin job, err: %v", err)
+		}
+		if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
+			aj.HandleConflict(fmt.Sprintf("Conflict when triggering %s, please try again later.", ajr.Name))
+			return
+		}
+		aj.HandleInternalServerError(fmt.Sprintf("%v", err))
+		return
+	}
+	if err := dao.SetAdminJobUUID(id, uuid); err != nil {
+		aj.HandleInternalServerError(fmt.Sprintf("%v", err))
+		return
+	}
+}
+
+func convertToAdminJobRep(job *common_models.AdminJob) (models.AdminJobRep, error) {
+	if job == nil {
+		return models.AdminJobRep{}, nil
+	}
+
+	AdminJobRep := models.AdminJobRep{
+		ID:           job.ID,
+		Name:         job.Name,
+		Kind:         job.Kind,
+		Status:       job.Status,
+		CreationTime: job.CreationTime,
+		UpdateTime:   job.UpdateTime,
+	}
+
+	if len(job.Cron) > 0 {
+		schedule := &models.ScheduleParam{}
+		if err := json.Unmarshal([]byte(job.Cron), &schedule); err != nil {
+			return models.AdminJobRep{}, err
+		}
+		AdminJobRep.Schedule = schedule
+	}
+	return AdminJobRep, nil
+}

src/core/api/harborapi_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/goharbor/harbor/src/common/job/test"
 	"github.com/goharbor/harbor/src/common/models"
 	testutils "github.com/goharbor/harbor/src/common/utils/test"
+	api_models "github.com/goharbor/harbor/src/core/api/models"
 	"github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/core/filter"
 	"github.com/goharbor/harbor/tests/apitests/apilib"
@@ -153,6 +154,7 @@ func init() {
 	beego.Router("/api/system/gc/:id", &GCAPI{}, "get:GetGC")
 	beego.Router("/api/system/gc/:id([0-9]+)/log", &GCAPI{}, "get:GetLog")
 	beego.Router("/api/system/gc/schedule", &GCAPI{}, "get:Get;put:Put;post:Post")
+	beego.Router("/api/system/scanAll/schedule", &ScanAllAPI{}, "get:Get;put:Put;post:Post")
 
 	beego.Router("/api/projects/:pid([0-9]+)/robots/", &RobotAPI{}, "post:Post;get:List")
 	beego.Router("/api/projects/:pid([0-9]+)/robots/:id([0-9]+)", &RobotAPI{}, "get:Get;put:Put;delete:Delete")
@@ -1183,7 +1185,7 @@ func (a testapi) DeleteMeta(authInfor usrInfo, projectID int64, name string) (in
 	return code, string(body), err
 }
 
-func (a testapi) AddGC(authInfor usrInfo, adminReq apilib.GCReq) (int, error) {
+func (a testapi) AddGC(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) {
 	_sling := sling.New().Post(a.basePath)
 
 	path := "/api/system/gc/schedule"
@@ -1200,12 +1202,42 @@ func (a testapi) AddGC(authInfor usrInfo, adminReq apilib.GCReq) (int, error) {
 	return httpStatusCode, err
 }
 
-func (a testapi) GCScheduleGet(authInfo usrInfo) (int, []apilib.AdminJob, error) {
+func (a testapi) GCScheduleGet(authInfo usrInfo) (int, api_models.AdminJobSchedule, error) {
 	_sling := sling.New().Get(a.basePath)
 	path := "/api/system/gc/schedule"
 	_sling = _sling.Path(path)
 	httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo)
-	var successPayLoad []apilib.AdminJob
+	var successPayLoad api_models.AdminJobSchedule
+	if 200 == httpStatusCode && nil == err {
+		err = json.Unmarshal(body, &successPayLoad)
+	}
+
+	return httpStatusCode, successPayLoad, err
+}
+
+func (a testapi) AddScanAll(authInfor usrInfo, adminReq apilib.AdminJobReq) (int, error) {
+	_sling := sling.New().Post(a.basePath)
+
+	path := "/api/system/scanAll/schedule"
+
+	_sling = _sling.Path(path)
+
+	// body params
+	_sling = _sling.BodyJSON(adminReq)
+	var httpStatusCode int
+	var err error
+
+	httpStatusCode, _, err = request(_sling, jsonAcceptHeader, authInfor)
+
+	return httpStatusCode, err
+}
+
+func (a testapi) ScanAllScheduleGet(authInfo usrInfo) (int, api_models.AdminJobSchedule, error) {
+	_sling := sling.New().Get(a.basePath)
+	path := "/api/system/scanAll/schedule"
+	_sling = _sling.Path(path)
+	httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo)
+	var successPayLoad api_models.AdminJobSchedule
 	if 200 == httpStatusCode && nil == err {
 		err = json.Unmarshal(body, &successPayLoad)
 	}

src/core/api/models/admin_job.go

@@ -42,14 +42,20 @@ const (
 	ScheduleNone = "None"
 )
 
-// GCReq holds request information for admin job
-type GCReq struct {
-	Schedule   *ScheduleParam         `json:"schedule"`
+// AdminJobReq holds request information for admin job
+type AdminJobReq struct {
+	AdminJobSchedule
+	Name       string                 `json:"name"`
 	Status     string                 `json:"status"`
 	ID         int64                  `json:"id"`
 	Parameters map[string]interface{} `json:"parameters"`
 }
 
+// AdminJobSchedule ...
+type AdminJobSchedule struct {
+	Schedule *ScheduleParam `json:"schedule"`
+}
+
 // ScheduleParam defines the parameter of schedule trigger
 type ScheduleParam struct {
 	// Daily, Weekly, Custom, Manual, None
@@ -58,62 +64,63 @@ type ScheduleParam struct {
 	Cron string `json:"cron"`
 }
 
-// GCRep holds the response of query gc
-type GCRep struct {
-	ID           int64          `json:"id"`
-	Name         string         `json:"job_name"`
-	Kind         string         `json:"job_kind"`
-	Schedule     *ScheduleParam `json:"schedule"`
-	Status       string         `json:"job_status"`
-	UUID         string         `json:"-"`
-	Deleted      bool           `json:"deleted"`
-	CreationTime time.Time      `json:"creation_time"`
-	UpdateTime   time.Time      `json:"update_time"`
+// AdminJobRep holds the response of query admin job
+type AdminJobRep struct {
+	AdminJobSchedule
+	ID           int64     `json:"id"`
+	Name         string    `json:"job_name"`
+	Kind         string    `json:"job_kind"`
+	Status       string    `json:"job_status"`
+	UUID         string    `json:"-"`
+	Deleted      bool      `json:"deleted"`
+	CreationTime time.Time `json:"creation_time"`
+	UpdateTime   time.Time `json:"update_time"`
 }
 
-// Valid validates the gc request
-func (gr *GCReq) Valid(v *validation.Validation) {
-	if gr.Schedule == nil {
+// Valid validates the schedule type of a admin job request.
+// Only scheduleHourly, ScheduleDaily, ScheduleWeekly, ScheduleCustom, ScheduleManual, ScheduleNone are accepted.
+func (ar *AdminJobReq) Valid(v *validation.Validation) {
+	if ar.Schedule == nil {
 		return
 	}
-	switch gr.Schedule.Type {
+	switch ar.Schedule.Type {
 	case ScheduleHourly, ScheduleDaily, ScheduleWeekly, ScheduleCustom:
-		if _, err := cron.Parse(gr.Schedule.Cron); err != nil {
-			v.SetError("cron", fmt.Sprintf("Invalid schedule trigger parameter cron: %s", gr.Schedule.Cron))
+		if _, err := cron.Parse(ar.Schedule.Cron); err != nil {
+			v.SetError("cron", fmt.Sprintf("Invalid schedule trigger parameter cron: %s", ar.Schedule.Cron))
 		}
 	case ScheduleManual, ScheduleNone:
 	default:
-		v.SetError("kind", fmt.Sprintf("Invalid schedule kind: %s", gr.Schedule.Type))
+		v.SetError("kind", fmt.Sprintf("Invalid schedule kind: %s", ar.Schedule.Type))
 	}
 }
 
 // ToJob converts request to a job recognized by job service.
-func (gr *GCReq) ToJob() *models.JobData {
+func (ar *AdminJobReq) ToJob() *models.JobData {
 	metadata := &models.JobMetadata{
-		JobKind: gr.JobKind(),
-		Cron:    gr.Schedule.Cron,
+		JobKind: ar.JobKind(),
+		Cron:    ar.Schedule.Cron,
 		// GC job must be unique ...
 		IsUnique: true,
 	}
 
 	jobData := &models.JobData{
-		Name:       job.ImageGC,
-		Parameters: gr.Parameters,
+		Name:       ar.Name,
+		Parameters: ar.Parameters,
 		Metadata:   metadata,
 		StatusHook: fmt.Sprintf("%s/service/notifications/jobs/adminjob/%d",
-			config.InternalCoreURL(), gr.ID),
+			config.InternalCoreURL(), ar.ID),
 	}
 	return jobData
 }
 
 // IsPeriodic ...
-func (gr *GCReq) IsPeriodic() bool {
-	return gr.JobKind() == job.JobKindPeriodic
+func (ar *AdminJobReq) IsPeriodic() bool {
+	return ar.JobKind() == job.JobKindPeriodic
 }
 
 // JobKind ...
-func (gr *GCReq) JobKind() string {
-	switch gr.Schedule.Type {
+func (ar *AdminJobReq) JobKind() string {
+	switch ar.Schedule.Type {
 	case ScheduleHourly, ScheduleDaily, ScheduleWeekly, ScheduleCustom:
 		return job.JobKindPeriodic
 	case ScheduleManual:
@@ -124,8 +131,8 @@ func (gr *GCReq) JobKind() string {
 }
 
 // CronString ...
-func (gr *GCReq) CronString() string {
-	str, err := json.Marshal(gr.Schedule)
+func (ar *AdminJobReq) CronString() string {
+	str, err := json.Marshal(ar.Schedule)
 	if err != nil {
 		log.Debugf("failed to marshal json error, %v", err)
 		return ""

src/core/api/models/admin_job_test.go

@@ -40,13 +40,17 @@ func TestMain(m *testing.M) {
 }
 
 func TestToJob(t *testing.T) {
-	schedule := &ScheduleParam{
-		Type: "Daily",
-		Cron: "20 3 0 * * *",
+
+	adminJobSchedule := AdminJobSchedule{
+		Schedule: &ScheduleParam{
+			Type: "Daily",
+			Cron: "20 3 0 * * *",
+		},
 	}
 
-	adminjob := &GCReq{
-		Schedule: schedule,
+	adminjob := &AdminJobReq{
+		Name:             common_job.ImageGC,
+		AdminJobSchedule: adminJobSchedule,
 	}
 
 	job := adminjob.ToJob()
@@ -56,12 +60,16 @@ func TestToJob(t *testing.T) {
 }
 
 func TestToJobManual(t *testing.T) {
-	schedule := &ScheduleParam{
-		Type: "Manual",
+
+	adminJobSchedule := AdminJobSchedule{
+		Schedule: &ScheduleParam{
+			Type: "Manual",
+		},
 	}
 
-	adminjob := &GCReq{
-		Schedule: schedule,
+	adminjob := &AdminJobReq{
+		AdminJobSchedule: adminJobSchedule,
+		Name:             common_job.ImageGC,
 	}
 
 	job := adminjob.ToJob()
@@ -70,13 +78,16 @@ func TestToJobManual(t *testing.T) {
 }
 
 func TestIsPeriodic(t *testing.T) {
-	schedule := &ScheduleParam{
-		Type: "Daily",
-		Cron: "20 3 0 * * *",
+
+	adminJobSchedule := AdminJobSchedule{
+		Schedule: &ScheduleParam{
+			Type: "Daily",
+			Cron: "20 3 0 * * *",
+		},
 	}
 
-	adminjob := &GCReq{
-		Schedule: schedule,
+	adminjob := &AdminJobReq{
+		AdminJobSchedule: adminJobSchedule,
 	}
 
 	isPeriodic := adminjob.IsPeriodic()
@@ -84,33 +95,44 @@ func TestIsPeriodic(t *testing.T) {
 }
 
 func TestJobKind(t *testing.T) {
-	schedule := &ScheduleParam{
-		Type: "Daily",
-		Cron: "20 3 0 * * *",
+
+	adminJobSchedule := AdminJobSchedule{
+		Schedule: &ScheduleParam{
+			Type: "Daily",
+			Cron: "20 3 0 * * *",
+		},
 	}
-	adminjob := &GCReq{
-		Schedule: schedule,
+
+	adminjob := &AdminJobReq{
+		AdminJobSchedule: adminJobSchedule,
 	}
+
 	kind := adminjob.JobKind()
 	assert.Equal(t, kind, "Periodic")
 
-	schedule1 := &ScheduleParam{
-		Type: "Manual",
+	adminJobSchedule1 := AdminJobSchedule{
+		Schedule: &ScheduleParam{
+			Type: "Manual",
+		},
 	}
-	adminjob1 := &GCReq{
-		Schedule: schedule1,
+	adminjob1 := &AdminJobReq{
+		AdminJobSchedule: adminJobSchedule1,
 	}
 	kind1 := adminjob1.JobKind()
 	assert.Equal(t, kind1, "Generic")
 }
 
 func TestCronString(t *testing.T) {
-	schedule := &ScheduleParam{
-		Type: "Daily",
-		Cron: "20 3 0 * * *",
+
+	adminJobSchedule := AdminJobSchedule{
+		Schedule: &ScheduleParam{
+			Type: "Daily",
+			Cron: "20 3 0 * * *",
+		},
 	}
-	adminjob := &GCReq{
-		Schedule: schedule,
+
+	adminjob := &AdminJobReq{
+		AdminJobSchedule: adminJobSchedule,
 	}
 	cronStr := adminjob.CronString()
 	assert.True(t, strings.EqualFold(cronStr, "{\"type\":\"Daily\",\"Cron\":\"20 3 0 * * *\"}"))

src/core/api/reg_gc.go

@@ -20,19 +20,13 @@ import (
 	"os"
 	"strconv"
 
-	"encoding/json"
-	"github.com/goharbor/harbor/src/common/dao"
-	common_http "github.com/goharbor/harbor/src/common/http"
 	common_job "github.com/goharbor/harbor/src/common/job"
-	common_models "github.com/goharbor/harbor/src/common/models"
-	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/core/api/models"
-	utils_core "github.com/goharbor/harbor/src/core/utils"
 )
 
-// GCAPI handles request of harbor admin...
+// GCAPI handles request of harbor GC...
 type GCAPI struct {
-	BaseController
+	AJAPI
 }
 
 // Prepare validates the URL and parms, it needs the system admin permission.
@@ -48,55 +42,44 @@ func (gc *GCAPI) Prepare() {
 	}
 }
 
-// Post ...
+// Post according to the request, it creates a cron schedule or a manual trigger for GC.
+// create a daily schedule for GC
+// 	{
+//  "schedule": {
+//    "type": "Daily",
+//    "cron": "0 0 0 * * *"
+//  }
+//	}
+// create a manual trigger for GC
+// 	{
+//  "schedule": {
+//    "type": "Manual"
+//  }
+//	}
 func (gc *GCAPI) Post() {
-	gr := models.GCReq{}
-	gc.DecodeJSONReqAndValidate(&gr)
-	gc.submitJob(&gr)
-	gc.Redirect(http.StatusCreated, strconv.FormatInt(gr.ID, 10))
+	ajr := models.AdminJobReq{}
+	gc.DecodeJSONReqAndValidate(&ajr)
+	ajr.Name = common_job.ImageGC
+	ajr.Parameters = map[string]interface{}{
+		"redis_url_reg": os.Getenv("_REDIS_URL_REG"),
+	}
+	gc.submit(&ajr)
+	gc.Redirect(http.StatusCreated, strconv.FormatInt(ajr.ID, 10))
 }
 
-// Put ...
+// Put handles GC cron schedule update/delete.
+// Request: delete the schedule of GC
+// 	{
+//  "schedule": {
+//    "type": "None",
+//    "cron": ""
+//  }
+//	}
 func (gc *GCAPI) Put() {
-	gr := models.GCReq{}
-	gc.DecodeJSONReqAndValidate(&gr)
-
-	if gr.Schedule.Type == models.ScheduleManual {
-		gc.HandleInternalServerError(fmt.Sprintf("Fail to update GC schedule as wrong schedule type: %s.", gr.Schedule.Type))
-		return
-	}
-
-	query := &common_models.AdminJobQuery{
-		Name: common_job.ImageGC,
-		Kind: common_job.JobKindPeriodic,
-	}
-	jobs, err := dao.GetAdminJobs(query)
-	if err != nil {
-		gc.HandleInternalServerError(fmt.Sprintf("%v", err))
-		return
-	}
-	if len(jobs) != 1 {
-		gc.HandleInternalServerError("Fail to update GC schedule, only one schedule is accepted.")
-		return
-	}
-
-	// stop the scheduled job and remove it.
-	if err = utils_core.GetJobServiceClient().PostAction(jobs[0].UUID, common_job.JobActionStop); err != nil {
-		if e, ok := err.(*common_http.Error); !ok || e.Code != http.StatusNotFound {
-			gc.HandleInternalServerError(fmt.Sprintf("%v", err))
-			return
-		}
-	}
-
-	if err = dao.DeleteAdminJob(jobs[0].ID); err != nil {
-		gc.HandleInternalServerError(fmt.Sprintf("%v", err))
-		return
-	}
-
-	// Set schedule to None means to cancel the schedule, won't add new job.
-	if gr.Schedule.Type != models.ScheduleNone {
-		gc.submitJob(&gr)
-	}
+	ajr := models.AdminJobReq{}
+	gc.DecodeJSONReqAndValidate(&ajr)
+	ajr.Name = common_job.ImageGC
+	gc.updateSchedule(ajr)
 }
 
 // GetGC ...
@@ -106,74 +89,17 @@ func (gc *GCAPI) GetGC() {
 		gc.HandleInternalServerError(fmt.Sprintf("need to specify gc id"))
 		return
 	}
-
-	jobs, err := dao.GetAdminJobs(&common_models.AdminJobQuery{
-		ID: id,
-	})
-
-	gcreps := []*models.GCRep{}
-	for _, job := range jobs {
-		gcrep, err := convertToGCRep(job)
-		if err != nil {
-			gc.HandleInternalServerError(fmt.Sprintf("failed to convert gc response: %v", err))
-			return
-		}
-		gcreps = append(gcreps, &gcrep)
-	}
-
-	if err != nil {
-		gc.HandleInternalServerError(fmt.Sprintf("failed to get admin jobs: %v", err))
-		return
-	}
-	gc.Data["json"] = gcreps
-	gc.ServeJSON()
+	gc.get(id)
 }
 
-// List ...
+// List returns the top 10 executions of GC which includes manual and cron.
 func (gc *GCAPI) List() {
-	jobs, err := dao.GetTop10AdminJobsOfName(common_job.ImageGC)
-	if err != nil {
-		gc.HandleInternalServerError(fmt.Sprintf("failed to get admin jobs: %v", err))
-		return
-	}
-	gcreps := []*models.GCRep{}
-	for _, job := range jobs {
-		gcrep, err := convertToGCRep(job)
-		if err != nil {
-			gc.HandleInternalServerError(fmt.Sprintf("failed to convert gc response: %v", err))
-			return
-		}
-		gcreps = append(gcreps, &gcrep)
-	}
-	gc.Data["json"] = gcreps
-	gc.ServeJSON()
+	gc.list(common_job.ImageGC)
 }
 
 // Get gets GC schedule ...
 func (gc *GCAPI) Get() {
-	jobs, err := dao.GetAdminJobs(&common_models.AdminJobQuery{
-		Name: common_job.ImageGC,
-		Kind: common_job.JobKindPeriodic,
-	})
-	if err != nil {
-		gc.HandleNotFound(fmt.Sprintf("failed to get admin jobs: %v", err))
-		return
-	}
-	if len(jobs) > 1 {
-		gc.HandleInternalServerError("Get more than one GC scheduled job, make sure there has only one.")
-		return
-	}
-	gcreps := []*models.GCRep{}
-	for _, job := range jobs {
-		gcrep, err := convertToGCRep(job)
-		if err != nil {
-			gc.HandleInternalServerError(fmt.Sprintf("failed to convert gc response: %v", err))
-			return
-		}
-		gcreps = append(gcreps, &gcrep)
-	}
-	gc.Data["json"] = gcreps
-	gc.ServeJSON()
+	gc.getSchedule(common_job.ImageGC)
 }
 
 // GetLog ...
@@ -183,108 +109,5 @@ func (gc *GCAPI) GetLog() {
 		gc.HandleBadRequest("invalid ID")
 		return
 	}
-	job, err := dao.GetAdminJob(id)
-	if err != nil {
-		log.Errorf("Failed to load job data for job: %d, error: %v", id, err)
-		gc.CustomAbort(http.StatusInternalServerError, "Failed to get Job data")
-	}
-	if job == nil {
-		log.Errorf("Failed to get admin job: %d", id)
-		gc.CustomAbort(http.StatusNotFound, "Failed to get Job")
-	}
-
-	logBytes, err := utils_core.GetJobServiceClient().GetJobLog(job.UUID)
-	if err != nil {
-		if httpErr, ok := err.(*common_http.Error); ok {
-			gc.RenderError(httpErr.Code, "")
-			log.Errorf(fmt.Sprintf("failed to get log of job %d: %d %s",
-				id, httpErr.Code, httpErr.Message))
-			return
-		}
-		gc.HandleInternalServerError(fmt.Sprintf("Failed to get job logs, uuid: %s, error: %v", job.UUID, err))
-		return
-	}
-	gc.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(logBytes)))
-	gc.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain")
-	_, err = gc.Ctx.ResponseWriter.Write(logBytes)
-	if err != nil {
-		gc.HandleInternalServerError(fmt.Sprintf("Failed to write job logs, uuid: %s, error: %v", job.UUID, err))
-	}
-}
-
-// submitJob submits a job to job service per request
-func (gc *GCAPI) submitJob(gr *models.GCReq) {
-	// cannot post multiple schedule for GC job.
-	if gr.IsPeriodic() {
-		jobs, err := dao.GetAdminJobs(&common_models.AdminJobQuery{
-			Name: common_job.ImageGC,
-			Kind: common_job.JobKindPeriodic,
-		})
-		if err != nil {
-			gc.HandleInternalServerError(fmt.Sprintf("failed to get admin jobs: %v", err))
-			return
-		}
-		if len(jobs) != 0 {
-			gc.HandleStatusPreconditionFailed("Fail to set schedule for GC as always had one, please delete it firstly then to re-schedule.")
-			return
-		}
-	}
-
-	id, err := dao.AddAdminJob(&common_models.AdminJob{
-		Name: common_job.ImageGC,
-		Kind: gr.JobKind(),
-		Cron: gr.CronString(),
-	})
-	if err != nil {
-		gc.HandleInternalServerError(fmt.Sprintf("%v", err))
-		return
-	}
-	gr.ID = id
-	gr.Parameters = map[string]interface{}{
-		"redis_url_reg": os.Getenv("_REDIS_URL_REG"),
-	}
-	job := gr.ToJob()
-	if err != nil {
-		gc.HandleInternalServerError(fmt.Sprintf("%v", err))
-		return
-	}
-
-	// submit job to jobservice
-	log.Debugf("submiting GC admin job to jobservice")
-	uuid, err := utils_core.GetJobServiceClient().SubmitJob(job)
-	if err != nil {
-		if err := dao.DeleteAdminJob(id); err != nil {
-			log.Debugf("Failed to delete admin job, err: %v", err)
-		}
-		gc.HandleInternalServerError(fmt.Sprintf("%v", err))
-		return
-	}
-	if err := dao.SetAdminJobUUID(id, uuid); err != nil {
-		gc.HandleInternalServerError(fmt.Sprintf("%v", err))
-		return
-	}
-}
-
-func convertToGCRep(job *common_models.AdminJob) (models.GCRep, error) {
-	if job == nil {
-		return models.GCRep{}, nil
-	}
-
-	gcrep := models.GCRep{
-		ID:           job.ID,
-		Name:         job.Name,
-		Kind:         job.Kind,
-		Status:       job.Status,
-		Deleted:      job.Deleted,
-		CreationTime: job.CreationTime,
-		UpdateTime:   job.UpdateTime,
-	}
-	if len(job.Cron) > 0 {
-		schedule := &models.ScheduleParam{}
-		if err := json.Unmarshal([]byte(job.Cron), &schedule); err != nil {
-			return models.GCRep{}, err
-		}
-		gcrep.Schedule = schedule
-	}
-	return gcrep, nil
+	gc.getLog(id)
 }

src/core/api/reg_gc_test.go

@@ -3,16 +3,13 @@ package api
 import (
 	"testing"
 
-	common_models "github.com/goharbor/harbor/src/common/models"
-	api_modes "github.com/goharbor/harbor/src/core/api/models"
 	"github.com/goharbor/harbor/tests/apitests/apilib"
 	"github.com/stretchr/testify/assert"
 )
 
-var adminJob001 apilib.GCReq
-var adminJob001schdeule apilib.ScheduleParam
+var adminJob001 apilib.AdminJobReq
 
-func TestAdminJobPost(t *testing.T) {
+func TestGCPost(t *testing.T) {
 
 	assert := assert.New(t)
 	apiTest := newHarborAPI()
@@ -27,7 +24,7 @@ func TestAdminJobPost(t *testing.T) {
 	}
 }
 
-func TestAdminJobGet(t *testing.T) {
+func TestGCGet(t *testing.T) {
 	assert := assert.New(t)
 	apiTest := newHarborAPI()
 
@@ -39,41 +36,3 @@ func TestAdminJobGet(t *testing.T) {
 		assert.Equal(200, code, "Get adminjob status should be 200")
 	}
 }
-
-func TestConvertToGCRep(t *testing.T) {
-	cases := []struct {
-		input    *common_models.AdminJob
-		expected api_modes.GCRep
-	}{
-		{
-			input:    nil,
-			expected: api_modes.GCRep{},
-		},
-		{
-			input: &common_models.AdminJob{
-				ID:      1,
-				Name:    "IMAGE_GC",
-				Kind:    "Generic",
-				Cron:    "{\"Type\":\"Daily\",\"Cron\":\"20 3 0 * * *\"}",
-				Status:  "pending",
-				Deleted: false,
-			},
-			expected: api_modes.GCRep{
-				ID:   1,
-				Name: "IMAGE_GC",
-				Kind: "Generic",
-				Schedule: &api_modes.ScheduleParam{
-					Type: "Daily",
-					Cron: "20 3 0 * * *",
-				},
-				Status:  "pending",
-				Deleted: false,
-			},
-		},
-	}
-
-	for _, c := range cases {
-		actual, _ := convertToGCRep(c.input)
-		assert.EqualValues(t, c.expected, actual)
-	}
-}

src/core/api/repository.go

@@ -1022,33 +1022,6 @@ func (ra *RepositoryAPI) VulnerabilityDetails() {
 	ra.ServeJSON()
 }
 
-// ScanAll handles the api to scan all images on Harbor.
-func (ra *RepositoryAPI) ScanAll() {
-	if !config.WithClair() {
-		log.Warningf("Harbor is not deployed with Clair, it's not possible to scan images.")
-		ra.RenderError(http.StatusServiceUnavailable, "")
-		return
-	}
-	if !ra.SecurityCtx.IsAuthenticated() {
-		ra.HandleUnauthorized()
-		return
-	}
-	if !ra.SecurityCtx.IsSysAdmin() {
-		ra.HandleForbidden(ra.SecurityCtx.GetUsername())
-		return
-	}
-	if err := coreutils.ScanAllImages(); err != nil {
-		log.Errorf("Failed triggering scan all images, error: %v", err)
-		if httpErr, ok := err.(*commonhttp.Error); ok && httpErr.Code == http.StatusConflict {
-			ra.HandleConflict("Conflict when triggering scan all images, please try again later.")
-			return
-		}
-		ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err))
-		return
-	}
-	ra.Ctx.ResponseWriter.WriteHeader(http.StatusAccepted)
-}
-
 func getSignatures(username, repository string) (map[string][]notary.Target, error) {
 	targets, err := notary.GetInternalTargets(config.InternalNotaryEndpoint(),
 		username, repository)

src/core/api/scan_all.go

@@ -0,0 +1,81 @@
+package api
+
+import (
+	"net/http"
+	"strconv"
+
+	common_job "github.com/goharbor/harbor/src/common/job"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/core/api/models"
+	"github.com/goharbor/harbor/src/core/config"
+)
+
+// ScanAllAPI handles request of scan all images...
+type ScanAllAPI struct {
+	AJAPI
+}
+
+// Prepare validates the URL and parms, it needs the system admin permission.
+func (sc *ScanAllAPI) Prepare() {
+	sc.BaseController.Prepare()
+	if !config.WithClair() {
+		log.Warningf("Harbor is not deployed with Clair, it's not possible to scan images.")
+		sc.RenderError(http.StatusServiceUnavailable, "")
+		return
+	}
+	if !sc.SecurityCtx.IsAuthenticated() {
+		sc.HandleUnauthorized()
+		return
+	}
+	if !sc.SecurityCtx.IsSysAdmin() {
+		sc.HandleForbidden(sc.SecurityCtx.GetUsername())
+		return
+	}
+}
+
+// Post according to the request, it creates a cron schedule or a manual trigger for scan all.
+// create a daily schedule for scan all
+// 	{
+//  "schedule": {
+//    "type": "Daily",
+//    "cron": "0 0 0 * * *"
+//  }
+//	}
+// create a manual trigger for scan all
+// 	{
+//  "schedule": {
+//    "type": "Manual"
+//  }
+//	}
+func (sc *ScanAllAPI) Post() {
+	ajr := models.AdminJobReq{}
+	sc.DecodeJSONReqAndValidate(&ajr)
+	ajr.Name = common_job.ImageScanAllJob
+	sc.submit(&ajr)
+	sc.Redirect(http.StatusCreated, strconv.FormatInt(ajr.ID, 10))
+}
+
+// Put handles scan all cron schedule update/delete.
+// Request: delete the schedule of scan all
+// 	{
+//  "schedule": {
+//    "type": "None",
+//    "cron": ""
+//  }
+//	}
+func (sc *ScanAllAPI) Put() {
+	ajr := models.AdminJobReq{}
+	sc.DecodeJSONReqAndValidate(&ajr)
+	ajr.Name = common_job.ImageScanAllJob
+	sc.updateSchedule(ajr)
+}
+
+// Get gets scan all schedule ...
+func (sc *ScanAllAPI) Get() {
+	sc.getSchedule(common_job.ImageScanAllJob)
+}
+
+// List returns the top 10 executions of scan all which includes manual and cron.
+func (sc *ScanAllAPI) List() {
+	sc.list(common_job.ImageScanAllJob)
+}

src/core/api/scan_all_test.go

@@ -0,0 +1,38 @@
+package api
+
+import (
+	"testing"
+
+	"github.com/goharbor/harbor/tests/apitests/apilib"
+	"github.com/stretchr/testify/assert"
+)
+
+var adminJob002 apilib.AdminJobReq
+
+func TestScanAllPost(t *testing.T) {
+
+	assert := assert.New(t)
+	apiTest := newHarborAPI()
+
+	// case 1: add a new scan all job
+	code, err := apiTest.AddScanAll(*admin, adminJob002)
+	if err != nil {
+		t.Error("Error occurred while add a scan all job", err.Error())
+		t.Log(err)
+	} else {
+		assert.Equal(200, code, "Add scan all status should be 200")
+	}
+}
+
+func TestScanAllGet(t *testing.T) {
+	assert := assert.New(t)
+	apiTest := newHarborAPI()
+
+	code, _, err := apiTest.ScanAllScheduleGet(*admin)
+	if err != nil {
+		t.Error("Error occurred while get a scan all job", err.Error())
+		t.Log(err)
+	} else {
+		assert.Equal(200, code, "Get scan all status should be 200")
+	}
+}

src/core/api/utils.go

@@ -29,7 +29,6 @@ import (
 	"github.com/goharbor/harbor/src/common/utils/registry"
 	"github.com/goharbor/harbor/src/common/utils/registry/auth"
 	"github.com/goharbor/harbor/src/core/config"
-	"github.com/goharbor/harbor/src/core/notifier"
 	"github.com/goharbor/harbor/src/core/promgr"
 	"github.com/goharbor/harbor/src/core/service/token"
 	coreutils "github.com/goharbor/harbor/src/core/utils"
@@ -312,9 +311,3 @@ func transformVulnerabilities(layerWithVuln *models.ClairLayerEnvelope) []*model
 	}
 	return res
 }
-
-// Watch the configuration changes.
-// Wrap the same method in common utils.
-func watchConfigChanges(cfg map[string]interface{}) error {
-	return notifier.WatchConfigChanges(cfg)
-}

src/core/main.go

@@ -34,7 +34,6 @@ import (
 	_ "github.com/goharbor/harbor/src/core/auth/uaa"
 	"github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/core/filter"
-	"github.com/goharbor/harbor/src/core/notifier"
 	"github.com/goharbor/harbor/src/core/proxy"
 	"github.com/goharbor/harbor/src/core/service/token"
 	"github.com/goharbor/harbor/src/replication/core"
@@ -113,11 +112,6 @@ func main() {
 		log.Fatalf("Failed to initialize API handlers with error: %s", err.Error())
 	}
 
-	// Subscribe the policy change topic.
-	if err = notifier.Subscribe(notifier.ScanAllPolicyTopic, &notifier.ScanPolicyNotificationHandler{}); err != nil {
-		log.Errorf("failed to subscribe scan all policy change topic: %v", err)
-	}
-
 	if config.WithClair() {
 		clairDB, err := config.ClairDB()
 		if err != nil {

src/core/notifier/config_watcher.go

@@ -1,40 +0,0 @@
-package notifier
-
-import (
-	"errors"
-
-	"github.com/goharbor/harbor/src/common/models"
-	"github.com/goharbor/harbor/src/common/utils"
-)
-
-// WatchConfigChanges is used to watch the configuration changes.
-func WatchConfigChanges(cfg map[string]interface{}) error {
-	if cfg == nil {
-		return errors.New("Empty configurations")
-	}
-
-	// Currently only watch the scan all policy change.
-	if v, ok := cfg[ScanAllPolicyTopic]; ok {
-		policyCfg := &models.ScanAllPolicy{}
-		if err := utils.ConvertMapToStruct(policyCfg, v); err != nil {
-			return err
-		}
-
-		policyNotification := ScanPolicyNotification{
-			Type:      policyCfg.Type,
-			DailyTime: 0,
-		}
-
-		if t, yes := policyCfg.Parm["daily_time"]; yes {
-			if dt, success := t.(float64); success {
-				policyNotification.DailyTime = (int64)(dt)
-			} else {
-				return errors.New("Invalid daily_time type")
-			}
-		}
-
-		return Publish(ScanAllPolicyTopic, policyNotification)
-	}
-
-	return nil
-}

src/core/notifier/config_watcher_test.go

@@ -1,57 +0,0 @@
-package notifier
-
-import (
-	"encoding/json"
-	"strconv"
-	"strings"
-	"testing"
-	"time"
-)
-
-var jsonText = `
-{
-"scan_all_policy": {
-   "type": "daily",
-      "parameter": {
-        "daily_time": <PLACE_HOLDER>
-      }
-  }
-}
-`
-
-func TestWatchConfiguration(t *testing.T) {
-	now := time.Now().UTC()
-	offset := (now.Hour()+1)*3600 + now.Minute()*60
-	jsonT := strings.Replace(jsonText, "<PLACE_HOLDER>", strconv.Itoa(offset), -1)
-	v := make(map[string]interface{})
-	if err := json.Unmarshal([]byte(jsonT), &v); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := WatchConfigChanges(v); err != nil {
-		if !strings.Contains(err.Error(), "No handlers registered") {
-			t.Fatal(err)
-		}
-	}
-}
-
-var jsonText2 = `
-{
-"scan_all_policy": {
-   "type": "none"
-  }
-}
-`
-
-func TestWatchConfiguration2(t *testing.T) {
-	v := make(map[string]interface{})
-	if err := json.Unmarshal([]byte(jsonText2), &v); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := WatchConfigChanges(v); err != nil {
-		if !strings.Contains(err.Error(), "No handlers registered") {
-			t.Fatal(err)
-		}
-	}
-}

src/core/notifier/scan_policy_notitification_handler.go

@@ -1,104 +0,0 @@
-package notifier
-
-import (
-	"errors"
-	"fmt"
-	"net/http"
-
-	"github.com/goharbor/harbor/src/common/dao"
-	common_http "github.com/goharbor/harbor/src/common/http"
-	"github.com/goharbor/harbor/src/common/job"
-	"github.com/goharbor/harbor/src/common/models"
-	common_utils "github.com/goharbor/harbor/src/common/utils"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/core/utils"
-)
-
-const (
-	// PolicyTypeDaily specify the policy type is "daily"
-	PolicyTypeDaily = "daily"
-	// PolicyTypeNone specify the policy type is "none"
-	PolicyTypeNone = "none"
-)
-
-// ScanPolicyNotification is defined for pass the policy change data.
-type ScanPolicyNotification struct {
-	// Type is used to keep the scan policy type: "none","daily" and "refresh".
-	Type string
-
-	// DailyTime is used when the type is 'daily', the offset with UTC time 00:00.
-	DailyTime int64
-}
-
-// ScanPolicyNotificationHandler is defined to handle the changes of scanning
-// policy.
-type ScanPolicyNotificationHandler struct{}
-
-// IsStateful to indicate this handler is stateful.
-func (s *ScanPolicyNotificationHandler) IsStateful() bool {
-	// Policy change should be done one by one.
-	return true
-}
-
-// Handle the policy change notification.
-func (s *ScanPolicyNotificationHandler) Handle(value interface{}) error {
-	notification, ok := value.(ScanPolicyNotification)
-	if !ok {
-		return errors.New("ScanPolicyNotificationHandler can not handle value with invalid type")
-	}
-
-	if notification.Type == PolicyTypeDaily {
-		if err := cancelScanAllJobs(); err != nil {
-			return fmt.Errorf("Failed to cancel scan_all jobs, error: %v", err)
-		}
-		h, m, s := common_utils.ParseOfftime(notification.DailyTime)
-		cron := fmt.Sprintf("%d %d %d * * *", s, m, h)
-		if err := utils.ScheduleScanAllImages(cron); err != nil {
-			return fmt.Errorf("Failed to schedule scan_all job, error: %v", err)
-		}
-	} else if notification.Type == PolicyTypeNone {
-		if err := cancelScanAllJobs(); err != nil {
-			return fmt.Errorf("Failed to cancel scan_all jobs, error: %v", err)
-		}
-	} else {
-		return fmt.Errorf("Notification type %s is not supported", notification.Type)
-	}
-
-	return nil
-}
-
-func cancelScanAllJobs(c ...job.Client) error {
-	var client job.Client
-	if c == nil || len(c) == 0 {
-		client = utils.GetJobServiceClient()
-	} else {
-		client = c[0]
-	}
-	q := &models.AdminJobQuery{
-		Name: job.ImageScanAllJob,
-		Kind: job.JobKindPeriodic,
-	}
-	jobs, err := dao.GetAdminJobs(q)
-	if err != nil {
-		log.Errorf("Failed to query sheduled scan_all jobs, error: %v", err)
-		return err
-	}
-	if len(jobs) > 1 {
-		log.Warningf("Got more than one scheduled scan_all jobs: %+v", jobs)
-	}
-	for _, j := range jobs {
-		if err := dao.DeleteAdminJob(j.ID); err != nil {
-			log.Warningf("Failed to delete scan_all job from DB, job ID: %d, job UUID: %s, error: %v", j.ID, j.UUID, err)
-		}
-		if err := client.PostAction(j.UUID, job.JobActionStop); err != nil {
-			if e, ok := err.(*common_http.Error); ok && e.Code == http.StatusNotFound {
-				log.Warningf("scan_all job not found on jobservice, UUID: %s, skip", j.UUID)
-			} else {
-				log.Errorf("Failed to stop scan_all job, UUID: %s, error: %v", j.UUID, e)
-				return e
-			}
-		}
-		log.Infof("scan_all job canceled, uuid: %s, id: %d", j.UUID, j.ID)
-	}
-	return nil
-}

src/core/notifier/scan_policy_notitification_handler_test.go

@@ -1,16 +0,0 @@
-package notifier
-
-import (
-	"github.com/stretchr/testify/assert"
-	"testing"
-)
-
-func TestScanPolicyNotificationHandler(t *testing.T) {
-	assert := assert.New(t)
-	s := &ScanPolicyNotificationHandler{}
-	assert.True(s.IsStateful())
-	err := s.Handle("")
-	if assert.NotNil(err) {
-		assert.Contains(err.Error(), "invalid type")
-	}
-}

src/core/router.go

@@ -72,7 +72,6 @@ func initRouters() {
 	beego.Router("/api/projects/:pid([0-9]+)/robots/:id([0-9]+)", &api.RobotAPI{}, "get:Get;put:Put;delete:Delete")
 
 	beego.Router("/api/repositories", &api.RepositoryAPI{}, "get:Get")
-	beego.Router("/api/repositories/scanAll", &api.RepositoryAPI{}, "post:ScanAll")
 	beego.Router("/api/repositories/*", &api.RepositoryAPI{}, "delete:Delete;put:Put")
 	beego.Router("/api/repositories/*/labels", &api.RepositoryLabelAPI{}, "get:GetOfRepository;post:AddToRepository")
 	beego.Router("/api/repositories/*/labels/:id([0-9]+)", &api.RepositoryLabelAPI{}, "delete:RemoveFromRepository")
@@ -94,6 +93,7 @@ func initRouters() {
 	beego.Router("/api/system/gc/:id", &api.GCAPI{}, "get:GetGC")
 	beego.Router("/api/system/gc/:id([0-9]+)/log", &api.GCAPI{}, "get:GetLog")
 	beego.Router("/api/system/gc/schedule", &api.GCAPI{}, "get:Get;put:Put;post:Post")
+	beego.Router("/api/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post")
 
 	beego.Router("/api/policies/replication/:id([0-9]+)", &api.RepPolicyAPI{})
 	beego.Router("/api/policies/replication", &api.RepPolicyAPI{}, "get:List")

src/core/utils/job.go

@@ -33,51 +33,6 @@ var (
 	jobServiceClient job.Client
 )
 
-// ScanAllImages scans all images of Harbor by submiting a scan all job to jobservice, and the job handler will call API
-// on the "core" service
-func ScanAllImages() error {
-	_, err := scanAll("")
-	return err
-}
-
-// ScheduleScanAllImages will schedule a scan all job based on the cron string, add append a record in admin job table.
-func ScheduleScanAllImages(cron string) error {
-	_, err := scanAll(cron)
-	return err
-}
-
-func scanAll(cron string, c ...job.Client) (string, error) {
-	var client job.Client
-	if c == nil || len(c) == 0 {
-		client = GetJobServiceClient()
-	} else {
-		client = c[0]
-	}
-	kind := job.JobKindGeneric
-	if len(cron) > 0 {
-		kind = job.JobKindPeriodic
-	}
-	meta := &jobmodels.JobMetadata{
-		JobKind:  kind,
-		IsUnique: true,
-		Cron:     cron,
-	}
-	id, err := dao.AddAdminJob(&models.AdminJob{
-		Name: job.ImageScanAllJob,
-		Kind: kind,
-	})
-	if err != nil {
-		return "", err
-	}
-	data := &jobmodels.JobData{
-		Name:       job.ImageScanAllJob,
-		Metadata:   meta,
-		StatusHook: fmt.Sprintf("%s/service/notifications/jobs/adminjob/%d", config.InternalCoreURL(), id),
-	}
-	log.Infof("scan_all job scheduled/triggered, cron string: '%s'", cron)
-	return client.SubmitJob(data)
-}
-
 // GetJobServiceClient returns the job service client instance.
 func GetJobServiceClient() job.Client {
 	cl.Lock()

tests/apitests/apilib/admin_job_req.go

@@ -22,8 +22,8 @@
 
 package apilib
 
-// GCReq holds request information for admin job
-type GCReq struct {
+// AdminJobReq holds request information for admin job
+type AdminJobReq struct {
 	Schedule *ScheduleParam `json:"schedule,omitempty"`
 	Status   string         `json:"status,omitempty"`
 	ID       int64          `json:"id,omitempty"`

tests/apitests/python/library/repository.py

@@ -107,11 +107,6 @@ class Repository(base.Base):
         base._assert_status_code(expect_status_code, status_code)
         return data
 
-    def scan_all_image_now(self, expect_status_code = 202, **kwargs):
-        client = self._get_client(**kwargs)
-        _, status_code, _ = client.repositories_scan_all_post_with_http_info()
-        base._assert_status_code(expect_status_code, status_code)
-
     def repository_should_exist(self, project_id, repo_name, **kwargs):
         repositories = self.get_repository(project_id, **kwargs)
         if is_repo_exist_in_project(repositories, repo_name) == False:

tests/apitests/python/library/system.py

@@ -71,14 +71,12 @@ class System(base.Base):
         base._assert_status_code(expect_status_code, status_code)
         return data
 
-    def set_gc_schedule(self, schedule_type = 'None', offtime = None, weekday = None, expect_status_code = 200, expect_response_body = None, **kwargs):
+    def set_gc_schedule(self, schedule_type = 'None', cron = None, expect_status_code = 200, expect_response_body = None, **kwargs):
         client = self._get_client(**kwargs)
-        gc_schedule = swagger_client.GCSchedule()
+        gc_schedule = swagger_client.AdminJobSchedule()
         gc_schedule.type = schedule_type
-        if offtime is not None:
-            gc_schedule.offtime = offtime
-        if weekday is not None:
-            gc_schedule.weekday = weekday
+        if cron is not None:
+            gc_schedule.cron = cron
         try:
             data, status_code, _ = client.system_gc_schedule_put_with_http_info(gc_schedule)
         except ApiException as e:
@@ -92,16 +90,14 @@ class System(base.Base):
         base._assert_status_code(expect_status_code, status_code)
         return data
 
-    def create_gc_schedule(self, schedule_type, offtime = None, weekday = None, expect_status_code = 201, expect_response_body = None, **kwargs):
+    def create_gc_schedule(self, schedule_type, cron = None, expect_status_code = 201, expect_response_body = None, **kwargs):
         client = self._get_client(**kwargs)
-        gcscheduleschedule = swagger_client.GCScheduleSchedule()
+        gcscheduleschedule = swagger_client.AdminJobScheduleObj()
         gcscheduleschedule.type = schedule_type
-        if offtime is not None:
-            gcscheduleschedule.offtime = offtime
-        if weekday is not None:
-            gcscheduleschedule.weekday = weekday
+        if cron is not None:
+            gcscheduleschedule.cron = cron
 
-        gc_schedule = swagger_client.GCSchedule(gcscheduleschedule)
+        gc_schedule = swagger_client.AdminJobSchedule(gcscheduleschedule)
         try:
             _, status_code, header = client.system_gc_schedule_post_with_http_info(gc_schedule)
         except ApiException as e:
@@ -115,6 +111,31 @@ class System(base.Base):
         base._assert_status_code(expect_status_code, status_code)
         return base._get_id_from_header(header)
 
+    def create_scan_all_schedule(self, schedule_type, cron = None, expect_status_code = 201, expect_response_body = None, **kwargs):
+        client = self._get_client(**kwargs)
+        scanschedule = swagger_client.AdminJobScheduleObj()
+        scanschedule.type = schedule_type
+        if cron is not None:
+            scanschedule.cron = cron
+
+        scan_all_schedule = swagger_client.AdminJobSchedule(scanschedule)
+        try:
+            _, status_code, header = client.system_scan_all_schedule_post_with_http_info(scan_all_schedule)
+        except ApiException as e:
+            if e.status == expect_status_code:
+                if expect_response_body is not None and e.body.strip() != expect_response_body.strip():
+                    raise Exception(r"Create Scan All schedule response body is not as expected {} actual status is {}.".format(expect_response_body.strip(), e.body.strip()))
+                else:
+                    return e.reason, e.body
+            else:
+                raise Exception(r"Create Scan All schedule result is not as expected {} actual status is {}.".format(expect_status_code, e.status))
+        base._assert_status_code(expect_status_code, status_code)
+        return base._get_id_from_header(header)
+
+    def scan_now(self, **kwargs):
+        scan_all_id = self.create_scan_all_schedule('Manual', **kwargs)
+        return scan_all_id
+
     def gc_now(self, **kwargs):
         gc_id = self.create_gc_schedule('Manual', **kwargs)
         return gc_id
@@ -125,9 +146,7 @@ class System(base.Base):
         while not (get_gc_status_finish):
             time.sleep(5)
             status = self.get_gc_status_by_id(gc_id, **kwargs)
-            if len(status) is not 1:
-                raise Exception(r"Get GC status count expected 1 actual count is {}.".format(len(status)))
-            if status[0].job_status == expected_gc_status:
+            if status.job_status == expected_gc_status:
                 get_gc_status_finish = True
             timeout_count = timeout_count - 1
 

tests/apitests/python/test_scan_all_images.py

@@ -4,6 +4,7 @@ import unittest
 from testutils import harbor_server
 from testutils import TEARDOWN
 from testutils import ADMIN_CLIENT
+from library.system import System
 from library.project import Project
 from library.user import User
 from library.repository import Repository
@@ -12,6 +13,9 @@ from library.repository import push_image_to_project
 class TestProjects(unittest.TestCase):
     @classmethod
     def setUp(self):
+        system = System()
+        self.system= system
+
         project = Project()
         self.project= project
 
@@ -86,7 +90,7 @@ class TestProjects(unittest.TestCase):
         TestProjects.repo_Luca_name, tag_Luca = push_image_to_project(project_Luca_name, harbor_server, user_Luca_name, user_common_password, image, src_tag)
 
         #4. Trigger scan all event;
-        self.repo.scan_all_image_now(**ADMIN_CLIENT)
+        self.system.scan_now(**ADMIN_CLIENT)
 
         #5. Check if image in project_Alice and another image in project_Luca were both scanned.
         self.repo.check_image_scan_result(TestProjects.repo_Alice_name, tag_Alice, expected_scan_status = "finished", **USER_ALICE_CLIENT)