From 4ad02de3482f80a8cfbb2198d1ed982f01695ba8 Mon Sep 17 00:00:00 2001 From: He Weiwei Date: Wed, 19 Feb 2020 12:21:54 +0800 Subject: [PATCH] Scan reorganize (#10735) * refactor(scan,scanner): move scan and scanner controllers to api pkg Signed-off-by: He Weiwei * feat(scan-all-job): move artifacts query from job to notification Move artifact query from scan all job to its notification handler to ensure that the components in pkg will not call controllers in api. Signed-off-by: He Weiwei --- src/api/scan/all_handler.go | 121 ++++++++++++++++++ .../scan => }/api/scan/base_controller.go | 2 +- .../api/scan/base_controller_test.go | 0 src/{pkg/scan => }/api/scan/controller.go | 0 src/{pkg/scan => }/api/scan/options.go | 0 .../scan => }/api/scanner/base_controller.go | 0 .../api/scanner/base_controller_test.go | 0 src/{pkg/scan => }/api/scanner/controller.go | 0 src/core/api/admin_job.go | 2 +- src/core/api/pro_scanner.go | 2 +- src/core/api/pro_scanner_test.go | 2 +- src/core/api/repository.go | 2 +- src/core/api/scan.go | 2 +- src/core/api/scan_all.go | 4 +- src/core/api/scan_test.go | 2 +- src/core/api/scanners.go | 2 +- src/core/api/scanners_test.go | 2 +- src/core/middlewares/vulnerable/handler.go | 5 +- .../notification/scan_image_handler.go | 2 +- .../notification/scan_image_handler_test.go | 2 +- .../service/notifications/admin/handler.go | 11 +- .../service/notifications/jobs/handler.go | 5 +- .../service/notifications/registry/handler.go | 7 +- src/pkg/scan/all/checkin.go | 47 ------- src/pkg/scan/all/job.go | 89 +------------ src/pkg/scan/api/scan/all_handler.go | 42 ------ src/pkg/scan/event/notification.go | 2 +- .../middleware/vulnerable/vulnerable.go | 7 +- 28 files changed, 158 insertions(+), 204 deletions(-) create mode 100644 src/api/scan/all_handler.go rename src/{pkg/scan => }/api/scan/base_controller.go (99%) rename src/{pkg/scan => }/api/scan/base_controller_test.go (100%) rename src/{pkg/scan => }/api/scan/controller.go (100%) rename src/{pkg/scan => }/api/scan/options.go (100%) rename src/{pkg/scan => }/api/scanner/base_controller.go (100%) rename src/{pkg/scan => }/api/scanner/base_controller_test.go (100%) rename src/{pkg/scan => }/api/scanner/controller.go (100%) delete mode 100644 src/pkg/scan/all/checkin.go delete mode 100644 src/pkg/scan/api/scan/all_handler.go diff --git a/src/api/scan/all_handler.go b/src/api/scan/all_handler.go new file mode 100644 index 000000000..5dcf0896a --- /dev/null +++ b/src/api/scan/all_handler.go @@ -0,0 +1,121 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "context" + + "github.com/goharbor/harbor/src/api/artifact" + "github.com/goharbor/harbor/src/api/repository" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/q" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/pkg/errors" +) + +// HandleCheckIn handles the check in data of the scan all job +func HandleCheckIn(ctx context.Context, checkIn string) { + if len(checkIn) == 0 { + // Nothing to handle, directly return + return + } + + batchSize := 50 + for repo := range fetchRepositories(ctx, batchSize) { + for artifact := range fetchArtifacts(ctx, repo.RepositoryID, batchSize) { + for _, tag := range artifact.Tags { + art := &v1.Artifact{ + NamespaceID: artifact.ProjectID, + Repository: repo.Name, + Tag: tag.Name, + Digest: artifact.Digest, + MimeType: artifact.ManifestMediaType, + } + if err := DefaultController.Scan(art, WithRequester(checkIn)); err != nil { + // Just logged + log.Error(errors.Wrap(err, "handle check in")) + } + } + } + } +} + +func fetchArtifacts(ctx context.Context, repositoryID int64, chunkSize int) <-chan *artifact.Artifact { + ch := make(chan *artifact.Artifact, chunkSize) + go func() { + defer close(ch) + + query := &q.Query{ + Keywords: map[string]interface{}{ + "repository_id": repositoryID, + }, + PageSize: int64(chunkSize), + PageNumber: 1, + } + + for { + _, artifacts, err := artifact.Ctl.List(ctx, query, &artifact.Option{WithTag: true}) + if err != nil { + log.Errorf("[scan all]: list artifacts failed, error: %v", err) + return + } + + for _, artifact := range artifacts { + ch <- artifact + } + + if len(artifacts) < chunkSize { + break + } + + query.PageNumber++ + } + + }() + + return ch +} + +func fetchRepositories(ctx context.Context, chunkSize int) <-chan *models.RepoRecord { + ch := make(chan *models.RepoRecord, chunkSize) + go func() { + defer close(ch) + + query := &q.Query{ + PageSize: int64(chunkSize), + PageNumber: 1, + } + + for { + _, repositories, err := repository.Ctl.List(ctx, query) + if err != nil { + log.Warningf("[scan all]: list repositories failed, error: %v", err) + break + } + + for _, repo := range repositories { + ch <- repo + } + + if len(repositories) < chunkSize { + break + } + + query.PageNumber++ + } + }() + return ch +} diff --git a/src/pkg/scan/api/scan/base_controller.go b/src/api/scan/base_controller.go similarity index 99% rename from src/pkg/scan/api/scan/base_controller.go rename to src/api/scan/base_controller.go index 7548d2638..7c391c441 100644 --- a/src/pkg/scan/api/scan/base_controller.go +++ b/src/api/scan/base_controller.go @@ -20,6 +20,7 @@ import ( "time" tk "github.com/docker/distribution/registry/auth/token" + sc "github.com/goharbor/harbor/src/api/scanner" cj "github.com/goharbor/harbor/src/common/job" jm "github.com/goharbor/harbor/src/common/job/models" "github.com/goharbor/harbor/src/common/rbac" @@ -32,7 +33,6 @@ import ( "github.com/goharbor/harbor/src/pkg/robot/model" sca "github.com/goharbor/harbor/src/pkg/scan" "github.com/goharbor/harbor/src/pkg/scan/all" - sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner" "github.com/goharbor/harbor/src/pkg/scan/dao/scan" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" "github.com/goharbor/harbor/src/pkg/scan/errs" diff --git a/src/pkg/scan/api/scan/base_controller_test.go b/src/api/scan/base_controller_test.go similarity index 100% rename from src/pkg/scan/api/scan/base_controller_test.go rename to src/api/scan/base_controller_test.go diff --git a/src/pkg/scan/api/scan/controller.go b/src/api/scan/controller.go similarity index 100% rename from src/pkg/scan/api/scan/controller.go rename to src/api/scan/controller.go diff --git a/src/pkg/scan/api/scan/options.go b/src/api/scan/options.go similarity index 100% rename from src/pkg/scan/api/scan/options.go rename to src/api/scan/options.go diff --git a/src/pkg/scan/api/scanner/base_controller.go b/src/api/scanner/base_controller.go similarity index 100% rename from src/pkg/scan/api/scanner/base_controller.go rename to src/api/scanner/base_controller.go diff --git a/src/pkg/scan/api/scanner/base_controller_test.go b/src/api/scanner/base_controller_test.go similarity index 100% rename from src/pkg/scan/api/scanner/base_controller_test.go rename to src/api/scanner/base_controller_test.go diff --git a/src/pkg/scan/api/scanner/controller.go b/src/api/scanner/controller.go similarity index 100% rename from src/pkg/scan/api/scanner/controller.go rename to src/api/scanner/controller.go diff --git a/src/core/api/admin_job.go b/src/core/api/admin_job.go index 43e663d2e..954644a5d 100644 --- a/src/core/api/admin_job.go +++ b/src/core/api/admin_job.go @@ -21,6 +21,7 @@ import ( "strconv" "time" + "github.com/goharbor/harbor/src/api/scan" "github.com/goharbor/harbor/src/common/dao" common_http "github.com/goharbor/harbor/src/common/http" common_job "github.com/goharbor/harbor/src/common/job" @@ -28,7 +29,6 @@ import ( "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/api/models" utils_core "github.com/goharbor/harbor/src/core/utils" - "github.com/goharbor/harbor/src/pkg/scan/api/scan" "github.com/pkg/errors" ) diff --git a/src/core/api/pro_scanner.go b/src/core/api/pro_scanner.go index 81a8b286a..4c14ace0f 100644 --- a/src/core/api/pro_scanner.go +++ b/src/core/api/pro_scanner.go @@ -15,9 +15,9 @@ package api import ( + "github.com/goharbor/harbor/src/api/scanner" "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/pkg/q" - "github.com/goharbor/harbor/src/pkg/scan/api/scanner" "github.com/pkg/errors" ) diff --git a/src/core/api/pro_scanner_test.go b/src/core/api/pro_scanner_test.go index 59b3bcb68..07d25ff4e 100644 --- a/src/core/api/pro_scanner_test.go +++ b/src/core/api/pro_scanner_test.go @@ -24,7 +24,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner" + sc "github.com/goharbor/harbor/src/api/scanner" "github.com/stretchr/testify/suite" ) diff --git a/src/core/api/repository.go b/src/core/api/repository.go index bdf687563..2fdabad14 100755 --- a/src/core/api/repository.go +++ b/src/core/api/repository.go @@ -27,6 +27,7 @@ import ( "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" + "github.com/goharbor/harbor/src/api/scan" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" commonhttp "github.com/goharbor/harbor/src/common/http" @@ -41,7 +42,6 @@ import ( "github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/immutabletag/match/rule" - "github.com/goharbor/harbor/src/pkg/scan/api/scan" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" "github.com/goharbor/harbor/src/pkg/signature/notary" notarymodel "github.com/goharbor/harbor/src/pkg/signature/notary/model" diff --git a/src/core/api/scan.go b/src/core/api/scan.go index 0a9a26eab..03da29cec 100644 --- a/src/core/api/scan.go +++ b/src/core/api/scan.go @@ -18,12 +18,12 @@ import ( "net/http" "strconv" + "github.com/goharbor/harbor/src/api/scan" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/utils" coreutils "github.com/goharbor/harbor/src/core/utils" "github.com/goharbor/harbor/src/jobservice/logger" - "github.com/goharbor/harbor/src/pkg/scan/api/scan" "github.com/goharbor/harbor/src/pkg/scan/errs" "github.com/goharbor/harbor/src/pkg/scan/report" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" diff --git a/src/core/api/scan_all.go b/src/core/api/scan_all.go index 31d4d76fa..717187cec 100644 --- a/src/core/api/scan_all.go +++ b/src/core/api/scan_all.go @@ -6,14 +6,14 @@ import ( "strconv" "strings" + "github.com/goharbor/harbor/src/api/scan" + "github.com/goharbor/harbor/src/api/scanner" common_job "github.com/goharbor/harbor/src/common/job" cm "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/core/api/models" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/pkg/q" "github.com/goharbor/harbor/src/pkg/scan/all" - "github.com/goharbor/harbor/src/pkg/scan/api/scan" - "github.com/goharbor/harbor/src/pkg/scan/api/scanner" "github.com/pkg/errors" ) diff --git a/src/core/api/scan_test.go b/src/core/api/scan_test.go index 60515b6ca..4d6f41152 100644 --- a/src/core/api/scan_test.go +++ b/src/core/api/scan_test.go @@ -19,9 +19,9 @@ import ( "net/http" "testing" + "github.com/goharbor/harbor/src/api/scan" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/pkg/scan/all" - "github.com/goharbor/harbor/src/pkg/scan/api/scan" dscan "github.com/goharbor/harbor/src/pkg/scan/dao/scan" "github.com/goharbor/harbor/src/pkg/scan/report" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" diff --git a/src/core/api/scanners.go b/src/core/api/scanners.go index ce88567a5..3c598fc70 100644 --- a/src/core/api/scanners.go +++ b/src/core/api/scanners.go @@ -18,8 +18,8 @@ import ( "fmt" "net/http" + s "github.com/goharbor/harbor/src/api/scanner" "github.com/goharbor/harbor/src/pkg/q" - s "github.com/goharbor/harbor/src/pkg/scan/api/scanner" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" "github.com/pkg/errors" ) diff --git a/src/core/api/scanners_test.go b/src/core/api/scanners_test.go index 231403c2a..9fa4432d1 100644 --- a/src/core/api/scanners_test.go +++ b/src/core/api/scanners_test.go @@ -21,8 +21,8 @@ import ( v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + sc "github.com/goharbor/harbor/src/api/scanner" "github.com/goharbor/harbor/src/pkg/q" - sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/src/core/middlewares/vulnerable/handler.go b/src/core/middlewares/vulnerable/handler.go index 605122136..0546638c2 100644 --- a/src/core/middlewares/vulnerable/handler.go +++ b/src/core/middlewares/vulnerable/handler.go @@ -17,15 +17,16 @@ package vulnerable import ( "net/http" + "net/http/httptest" + + sc "github.com/goharbor/harbor/src/api/scan" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/middlewares/util" - sc "github.com/goharbor/harbor/src/pkg/scan/api/scan" "github.com/goharbor/harbor/src/pkg/scan/report" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" "github.com/goharbor/harbor/src/pkg/scan/vuln" "github.com/pkg/errors" - "net/http/httptest" ) type vulnerableHandler struct { diff --git a/src/core/notifier/handler/notification/scan_image_handler.go b/src/core/notifier/handler/notification/scan_image_handler.go index defaedf38..3537535c7 100644 --- a/src/core/notifier/handler/notification/scan_image_handler.go +++ b/src/core/notifier/handler/notification/scan_image_handler.go @@ -3,12 +3,12 @@ package notification import ( "time" + "github.com/goharbor/harbor/src/api/scan" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/notifier/model" "github.com/goharbor/harbor/src/pkg/notification" - "github.com/goharbor/harbor/src/pkg/scan/api/scan" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" "github.com/pkg/errors" ) diff --git a/src/core/notifier/handler/notification/scan_image_handler_test.go b/src/core/notifier/handler/notification/scan_image_handler_test.go index bd6f9a099..fde7c725e 100644 --- a/src/core/notifier/handler/notification/scan_image_handler_test.go +++ b/src/core/notifier/handler/notification/scan_image_handler_test.go @@ -6,6 +6,7 @@ import ( "github.com/goharbor/harbor/src/pkg/scan/all" + sc "github.com/goharbor/harbor/src/api/scan" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/notifier" @@ -14,7 +15,6 @@ import ( "github.com/goharbor/harbor/src/pkg/notification" nm "github.com/goharbor/harbor/src/pkg/notification/model" "github.com/goharbor/harbor/src/pkg/notification/policy" - sc "github.com/goharbor/harbor/src/pkg/scan/api/scan" "github.com/goharbor/harbor/src/pkg/scan/dao/scan" "github.com/goharbor/harbor/src/pkg/scan/report" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" diff --git a/src/core/service/notifications/admin/handler.go b/src/core/service/notifications/admin/handler.go index a9d8e7637..a7a50b72e 100644 --- a/src/core/service/notifications/admin/handler.go +++ b/src/core/service/notifications/admin/handler.go @@ -15,16 +15,19 @@ package admin import ( + "context" "encoding/json" - "github.com/goharbor/harbor/src/core/service/notifications" + o "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/api/scan" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/job" job_model "github.com/goharbor/harbor/src/common/job/models" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/service/notifications" + "github.com/goharbor/harbor/src/internal/orm" j "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/pkg/scan/api/scan" ) var statusMap = map[string]string{ @@ -104,7 +107,7 @@ func (h *Handler) HandleAdminJob() { } // For scan all job - if h.jobName == job.ImageScanAllJob { - scan.HandleCheckIn(h.checkIn) + if h.jobName == job.ImageScanAllJob && h.checkIn != "" { + go scan.HandleCheckIn(orm.NewContext(context.TODO(), o.NewOrm()), h.checkIn) } } diff --git a/src/core/service/notifications/jobs/handler.go b/src/core/service/notifications/jobs/handler.go index b142855c1..2be15f08a 100755 --- a/src/core/service/notifications/jobs/handler.go +++ b/src/core/service/notifications/jobs/handler.go @@ -16,9 +16,11 @@ package jobs import ( "encoding/json" - "github.com/goharbor/harbor/src/core/service/notifications" "time" + "github.com/goharbor/harbor/src/core/service/notifications" + + "github.com/goharbor/harbor/src/api/scan" "github.com/goharbor/harbor/src/common/job" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/log" @@ -27,7 +29,6 @@ import ( "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/retention" sc "github.com/goharbor/harbor/src/pkg/scan" - "github.com/goharbor/harbor/src/pkg/scan/api/scan" "github.com/goharbor/harbor/src/replication" "github.com/goharbor/harbor/src/replication/operation/hook" "github.com/goharbor/harbor/src/replication/policy/scheduler" diff --git a/src/core/service/notifications/registry/handler.go b/src/core/service/notifications/registry/handler.go index dffea745d..a9f0e6087 100755 --- a/src/core/service/notifications/registry/handler.go +++ b/src/core/service/notifications/registry/handler.go @@ -16,12 +16,15 @@ package registry import ( "encoding/json" - "github.com/goharbor/harbor/src/core/service/notifications" "regexp" "strconv" "strings" "time" + "github.com/goharbor/harbor/src/core/service/notifications" + + "github.com/goharbor/harbor/src/api/scan" + "github.com/goharbor/harbor/src/api/scanner" "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils" @@ -29,8 +32,6 @@ import ( "github.com/goharbor/harbor/src/core/config" notifierEvt "github.com/goharbor/harbor/src/core/notifier/event" coreutils "github.com/goharbor/harbor/src/core/utils" - "github.com/goharbor/harbor/src/pkg/scan/api/scan" - "github.com/goharbor/harbor/src/pkg/scan/api/scanner" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" "github.com/goharbor/harbor/src/replication" "github.com/goharbor/harbor/src/replication/adapter" diff --git a/src/pkg/scan/all/checkin.go b/src/pkg/scan/all/checkin.go deleted file mode 100644 index 23041da8e..000000000 --- a/src/pkg/scan/all/checkin.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package all - -import ( - "encoding/json" - - v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" - "github.com/pkg/errors" -) - -// CheckInData is designed for checking the data generated by the scan all job. -type CheckInData struct { - Artifacts []*v1.Artifact `json:"artifacts"` - Requester string `json:"requester"` -} - -// ToJSON marshals `CheckInData` to JSON str -func (c *CheckInData) ToJSON() (string, error) { - data, err := json.Marshal(c) - if err != nil { - return "", errors.Wrap(err, "ToJSON : CheckInData") - } - - return string(data), nil -} - -// FromJSON unmarshal give bytes to `CheckInData` -func (c *CheckInData) FromJSON(data []byte) error { - if err := json.Unmarshal(data, c); err != nil { - return errors.Wrap(err, "FromJSON : CheckInData") - } - - return nil -} diff --git a/src/pkg/scan/all/job.go b/src/pkg/scan/all/job.go index fb4bff423..68912be7d 100644 --- a/src/pkg/scan/all/job.go +++ b/src/pkg/scan/all/job.go @@ -15,12 +15,7 @@ package all import ( - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/jobservice/job" - "github.com/goharbor/harbor/src/pkg/art" - "github.com/goharbor/harbor/src/pkg/q" - v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" "github.com/pkg/errors" ) @@ -63,88 +58,8 @@ func (sa *Job) Run(ctx job.Context, params job.Parameters) error { // No need to check error any more as it has been checked in job validation. requester, _ := parseAJID(params) - // List all the repositories of registry - // TODO: REPLACE DAO WITH CORRESPONDING MANAGER OR CTL - repos, err := dao.GetRepositories() - if err != nil { - err = errors.Wrap(err, "list repositories : scan all job") - logger.Error(err) - return err - } - logger.Infof("Found %d repositories", len(repos)) - - // Initialize tokens - tokens := make(chan bool, maxProcessors) - for i := 0; i < maxProcessors; i++ { - // Assign tokens at first - tokens <- true - } - - // Get the tags under the repository - for _, r := range repos { - // Get token first - <-tokens - - go func(repo *models.RepoRecord) { - defer func() { - // Return the token when process ending - tokens <- true - }() - - logger.Infof("Scan artifacts under repository: %s", repo.Name) - - // Query artifacts under the repository - query := &q.Query{ - Keywords: make(map[string]interface{}), - } - query.Keywords["repo"] = repo.Name - - al, err := art.DefaultController.List(query) - if err != nil { - logger.Errorf("Failed to get tags for repo: %s, error: %v", repo.Name, err) - return - } - - if len(al) > 0 { - // Check in the data - arts := make([]*v1.Artifact, 0) - - for _, a := range al { - artf := &v1.Artifact{ - NamespaceID: repo.ProjectID, - Repository: repo.Name, - Tag: a.Tag, - Digest: a.Digest, - MimeType: v1.MimeTypeDockerArtifact, // default - } - - arts = append(arts, artf) - } - - logger.Infof("Found %d artifacts under repository %s", len(arts), repo.Name) - - ck := &CheckInData{ - Artifacts: arts, - Requester: requester, - } - - jsn, err := ck.ToJSON() - if err != nil { - logger.Error(errors.Wrap(err, "scan all job")) - return - } - - if err := ctx.Checkin(jsn); err != nil { - logger.Error(errors.Wrap(err, "check in data: scan all job")) - } - - logger.Infof("Check in scanning artifacts for repository: %s", repo.Name) - // Debug more - logger.Debugf("Check in: %s\n", jsn) - } else { - logger.Infof("No scanning artifacts found under repository: %s", repo.Name) - } - }(r) + if err := ctx.Checkin(requester); err != nil { + logger.Error(errors.Wrap(err, "check in data: scan all job")) } return nil diff --git a/src/pkg/scan/api/scan/all_handler.go b/src/pkg/scan/api/scan/all_handler.go deleted file mode 100644 index c10c17dd6..000000000 --- a/src/pkg/scan/api/scan/all_handler.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scan - -import ( - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/pkg/scan/all" - "github.com/pkg/errors" -) - -// HandleCheckIn handles the check in data of the scan all job -func HandleCheckIn(checkIn string) { - if len(checkIn) == 0 { - // Nothing to handle, directly return - return - } - - ck := &all.CheckInData{} - if err := ck.FromJSON([]byte(checkIn)); err != nil { - log.Error(errors.Wrap(err, "handle check in")) - } - - // Start to scan the artifacts - for _, art := range ck.Artifacts { - if err := DefaultController.Scan(art, WithRequester(ck.Requester)); err != nil { - // Just logged - log.Error(errors.Wrap(err, "handle check in")) - } - } -} diff --git a/src/pkg/scan/event/notification.go b/src/pkg/scan/event/notification.go index 523c53897..3db66d9a1 100644 --- a/src/pkg/scan/event/notification.go +++ b/src/pkg/scan/event/notification.go @@ -15,12 +15,12 @@ package event import ( + "github.com/goharbor/harbor/src/api/scan" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/notifier" "github.com/goharbor/harbor/src/core/notifier/model" "github.com/goharbor/harbor/src/pkg/art" "github.com/goharbor/harbor/src/pkg/q" - "github.com/goharbor/harbor/src/pkg/scan/api/scan" "github.com/pkg/errors" ) diff --git a/src/server/middleware/vulnerable/vulnerable.go b/src/server/middleware/vulnerable/vulnerable.go index 448df3492..ac23c7b0c 100644 --- a/src/server/middleware/vulnerable/vulnerable.go +++ b/src/server/middleware/vulnerable/vulnerable.go @@ -1,18 +1,19 @@ package vulnerable import ( + "net/http" + "net/http/httptest" + + sc "github.com/goharbor/harbor/src/api/scan" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/log" internal_errors "github.com/goharbor/harbor/src/internal/error" - sc "github.com/goharbor/harbor/src/pkg/scan/api/scan" "github.com/goharbor/harbor/src/pkg/scan/report" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" "github.com/goharbor/harbor/src/pkg/scan/vuln" serror "github.com/goharbor/harbor/src/server/error" "github.com/goharbor/harbor/src/server/middleware" "github.com/pkg/errors" - "net/http" - "net/http/httptest" ) // Middleware handle docker pull vulnerable check