Scan reorganize (#10735)

* refactor(scan,scanner): move scan and scanner controllers to api pkg

Signed-off-by: He Weiwei <hweiwei@vmware.com>

* feat(scan-all-job): move artifacts query from job to notification

Move artifact query from scan all job to its notification handler to
ensure that the components in pkg will not call controllers in api.

Signed-off-by: He Weiwei <hweiwei@vmware.com>
This commit is contained in:
He Weiwei 2020-02-19 12:21:54 +08:00 committed by GitHub
parent 384086c5ad
commit 4ad02de348
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 158 additions and 204 deletions

121
src/api/scan/all_handler.go Normal file
View File

@ -0,0 +1,121 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"context"
"github.com/goharbor/harbor/src/api/artifact"
"github.com/goharbor/harbor/src/api/repository"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/q"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/pkg/errors"
)
// HandleCheckIn handles the check in data of the scan all job
func HandleCheckIn(ctx context.Context, checkIn string) {
if len(checkIn) == 0 {
// Nothing to handle, directly return
return
}
batchSize := 50
for repo := range fetchRepositories(ctx, batchSize) {
for artifact := range fetchArtifacts(ctx, repo.RepositoryID, batchSize) {
for _, tag := range artifact.Tags {
art := &v1.Artifact{
NamespaceID: artifact.ProjectID,
Repository: repo.Name,
Tag: tag.Name,
Digest: artifact.Digest,
MimeType: artifact.ManifestMediaType,
}
if err := DefaultController.Scan(art, WithRequester(checkIn)); err != nil {
// Just logged
log.Error(errors.Wrap(err, "handle check in"))
}
}
}
}
}
func fetchArtifacts(ctx context.Context, repositoryID int64, chunkSize int) <-chan *artifact.Artifact {
ch := make(chan *artifact.Artifact, chunkSize)
go func() {
defer close(ch)
query := &q.Query{
Keywords: map[string]interface{}{
"repository_id": repositoryID,
},
PageSize: int64(chunkSize),
PageNumber: 1,
}
for {
_, artifacts, err := artifact.Ctl.List(ctx, query, &artifact.Option{WithTag: true})
if err != nil {
log.Errorf("[scan all]: list artifacts failed, error: %v", err)
return
}
for _, artifact := range artifacts {
ch <- artifact
}
if len(artifacts) < chunkSize {
break
}
query.PageNumber++
}
}()
return ch
}
func fetchRepositories(ctx context.Context, chunkSize int) <-chan *models.RepoRecord {
ch := make(chan *models.RepoRecord, chunkSize)
go func() {
defer close(ch)
query := &q.Query{
PageSize: int64(chunkSize),
PageNumber: 1,
}
for {
_, repositories, err := repository.Ctl.List(ctx, query)
if err != nil {
log.Warningf("[scan all]: list repositories failed, error: %v", err)
break
}
for _, repo := range repositories {
ch <- repo
}
if len(repositories) < chunkSize {
break
}
query.PageNumber++
}
}()
return ch
}

View File

@ -20,6 +20,7 @@ import (
"time"
tk "github.com/docker/distribution/registry/auth/token"
sc "github.com/goharbor/harbor/src/api/scanner"
cj "github.com/goharbor/harbor/src/common/job"
jm "github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/common/rbac"
@ -32,7 +33,6 @@ import (
"github.com/goharbor/harbor/src/pkg/robot/model"
sca "github.com/goharbor/harbor/src/pkg/scan"
"github.com/goharbor/harbor/src/pkg/scan/all"
sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/errs"

View File

@ -21,6 +21,7 @@ import (
"strconv"
"time"
"github.com/goharbor/harbor/src/api/scan"
"github.com/goharbor/harbor/src/common/dao"
common_http "github.com/goharbor/harbor/src/common/http"
common_job "github.com/goharbor/harbor/src/common/job"
@ -28,7 +29,6 @@ import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/api/models"
utils_core "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/pkg/errors"
)

View File

@ -15,9 +15,9 @@
package api
import (
"github.com/goharbor/harbor/src/api/scanner"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/api/scanner"
"github.com/pkg/errors"
)

View File

@ -24,7 +24,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner"
sc "github.com/goharbor/harbor/src/api/scanner"
"github.com/stretchr/testify/suite"
)

View File

@ -27,6 +27,7 @@ import (
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/api/scan"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
commonhttp "github.com/goharbor/harbor/src/common/http"
@ -41,7 +42,6 @@ import (
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/art"
"github.com/goharbor/harbor/src/pkg/immutabletag/match/rule"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/signature/notary"
notarymodel "github.com/goharbor/harbor/src/pkg/signature/notary/model"

View File

@ -18,12 +18,12 @@ import (
"net/http"
"strconv"
"github.com/goharbor/harbor/src/api/scan"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/utils"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/pkg/scan/errs"
"github.com/goharbor/harbor/src/pkg/scan/report"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"

View File

@ -6,14 +6,14 @@ import (
"strconv"
"strings"
"github.com/goharbor/harbor/src/api/scan"
"github.com/goharbor/harbor/src/api/scanner"
common_job "github.com/goharbor/harbor/src/common/job"
cm "github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/all"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/pkg/scan/api/scanner"
"github.com/pkg/errors"
)

View File

@ -19,9 +19,9 @@ import (
"net/http"
"testing"
"github.com/goharbor/harbor/src/api/scan"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/scan/all"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
dscan "github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/report"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"

View File

@ -18,8 +18,8 @@ import (
"fmt"
"net/http"
s "github.com/goharbor/harbor/src/api/scanner"
"github.com/goharbor/harbor/src/pkg/q"
s "github.com/goharbor/harbor/src/pkg/scan/api/scanner"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/pkg/errors"
)

View File

@ -21,8 +21,8 @@ import (
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
sc "github.com/goharbor/harbor/src/api/scanner"
"github.com/goharbor/harbor/src/pkg/q"
sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"

View File

@ -17,15 +17,16 @@ package vulnerable
import (
"net/http"
"net/http/httptest"
sc "github.com/goharbor/harbor/src/api/scan"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/util"
sc "github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/pkg/scan/report"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
"github.com/pkg/errors"
"net/http/httptest"
)
type vulnerableHandler struct {

View File

@ -3,12 +3,12 @@ package notification
import (
"time"
"github.com/goharbor/harbor/src/api/scan"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/notifier/model"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/pkg/errors"
)

View File

@ -6,6 +6,7 @@ import (
"github.com/goharbor/harbor/src/pkg/scan/all"
sc "github.com/goharbor/harbor/src/api/scan"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/notifier"
@ -14,7 +15,6 @@ import (
"github.com/goharbor/harbor/src/pkg/notification"
nm "github.com/goharbor/harbor/src/pkg/notification/model"
"github.com/goharbor/harbor/src/pkg/notification/policy"
sc "github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/report"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"

View File

@ -15,16 +15,19 @@
package admin
import (
"context"
"encoding/json"
"github.com/goharbor/harbor/src/core/service/notifications"
o "github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/api/scan"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/job"
job_model "github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/service/notifications"
"github.com/goharbor/harbor/src/internal/orm"
j "github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
)
var statusMap = map[string]string{
@ -104,7 +107,7 @@ func (h *Handler) HandleAdminJob() {
}
// For scan all job
if h.jobName == job.ImageScanAllJob {
scan.HandleCheckIn(h.checkIn)
if h.jobName == job.ImageScanAllJob && h.checkIn != "" {
go scan.HandleCheckIn(orm.NewContext(context.TODO(), o.NewOrm()), h.checkIn)
}
}

View File

@ -16,9 +16,11 @@ package jobs
import (
"encoding/json"
"github.com/goharbor/harbor/src/core/service/notifications"
"time"
"github.com/goharbor/harbor/src/core/service/notifications"
"github.com/goharbor/harbor/src/api/scan"
"github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
@ -27,7 +29,6 @@ import (
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/pkg/retention"
sc "github.com/goharbor/harbor/src/pkg/scan"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/operation/hook"
"github.com/goharbor/harbor/src/replication/policy/scheduler"

View File

@ -16,12 +16,15 @@ package registry
import (
"encoding/json"
"github.com/goharbor/harbor/src/core/service/notifications"
"regexp"
"strconv"
"strings"
"time"
"github.com/goharbor/harbor/src/core/service/notifications"
"github.com/goharbor/harbor/src/api/scan"
"github.com/goharbor/harbor/src/api/scanner"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils"
@ -29,8 +32,6 @@ import (
"github.com/goharbor/harbor/src/core/config"
notifierEvt "github.com/goharbor/harbor/src/core/notifier/event"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/pkg/scan/api/scanner"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/adapter"

View File

@ -1,47 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package all
import (
"encoding/json"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/pkg/errors"
)
// CheckInData is designed for checking the data generated by the scan all job.
type CheckInData struct {
Artifacts []*v1.Artifact `json:"artifacts"`
Requester string `json:"requester"`
}
// ToJSON marshals `CheckInData` to JSON str
func (c *CheckInData) ToJSON() (string, error) {
data, err := json.Marshal(c)
if err != nil {
return "", errors.Wrap(err, "ToJSON : CheckInData")
}
return string(data), nil
}
// FromJSON unmarshal give bytes to `CheckInData`
func (c *CheckInData) FromJSON(data []byte) error {
if err := json.Unmarshal(data, c); err != nil {
return errors.Wrap(err, "FromJSON : CheckInData")
}
return nil
}

View File

@ -15,12 +15,7 @@
package all
import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/art"
"github.com/goharbor/harbor/src/pkg/q"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/pkg/errors"
)
@ -63,90 +58,10 @@ func (sa *Job) Run(ctx job.Context, params job.Parameters) error {
// No need to check error any more as it has been checked in job validation.
requester, _ := parseAJID(params)
// List all the repositories of registry
// TODO: REPLACE DAO WITH CORRESPONDING MANAGER OR CTL
repos, err := dao.GetRepositories()
if err != nil {
err = errors.Wrap(err, "list repositories : scan all job")
logger.Error(err)
return err
}
logger.Infof("Found %d repositories", len(repos))
// Initialize tokens
tokens := make(chan bool, maxProcessors)
for i := 0; i < maxProcessors; i++ {
// Assign tokens at first
tokens <- true
}
// Get the tags under the repository
for _, r := range repos {
// Get token first
<-tokens
go func(repo *models.RepoRecord) {
defer func() {
// Return the token when process ending
tokens <- true
}()
logger.Infof("Scan artifacts under repository: %s", repo.Name)
// Query artifacts under the repository
query := &q.Query{
Keywords: make(map[string]interface{}),
}
query.Keywords["repo"] = repo.Name
al, err := art.DefaultController.List(query)
if err != nil {
logger.Errorf("Failed to get tags for repo: %s, error: %v", repo.Name, err)
return
}
if len(al) > 0 {
// Check in the data
arts := make([]*v1.Artifact, 0)
for _, a := range al {
artf := &v1.Artifact{
NamespaceID: repo.ProjectID,
Repository: repo.Name,
Tag: a.Tag,
Digest: a.Digest,
MimeType: v1.MimeTypeDockerArtifact, // default
}
arts = append(arts, artf)
}
logger.Infof("Found %d artifacts under repository %s", len(arts), repo.Name)
ck := &CheckInData{
Artifacts: arts,
Requester: requester,
}
jsn, err := ck.ToJSON()
if err != nil {
logger.Error(errors.Wrap(err, "scan all job"))
return
}
if err := ctx.Checkin(jsn); err != nil {
if err := ctx.Checkin(requester); err != nil {
logger.Error(errors.Wrap(err, "check in data: scan all job"))
}
logger.Infof("Check in scanning artifacts for repository: %s", repo.Name)
// Debug more
logger.Debugf("Check in: %s\n", jsn)
} else {
logger.Infof("No scanning artifacts found under repository: %s", repo.Name)
}
}(r)
}
return nil
}

View File

@ -1,42 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/scan/all"
"github.com/pkg/errors"
)
// HandleCheckIn handles the check in data of the scan all job
func HandleCheckIn(checkIn string) {
if len(checkIn) == 0 {
// Nothing to handle, directly return
return
}
ck := &all.CheckInData{}
if err := ck.FromJSON([]byte(checkIn)); err != nil {
log.Error(errors.Wrap(err, "handle check in"))
}
// Start to scan the artifacts
for _, art := range ck.Artifacts {
if err := DefaultController.Scan(art, WithRequester(ck.Requester)); err != nil {
// Just logged
log.Error(errors.Wrap(err, "handle check in"))
}
}
}

View File

@ -15,12 +15,12 @@
package event
import (
"github.com/goharbor/harbor/src/api/scan"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/notifier"
"github.com/goharbor/harbor/src/core/notifier/model"
"github.com/goharbor/harbor/src/pkg/art"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/pkg/errors"
)

View File

@ -1,18 +1,19 @@
package vulnerable
import (
"net/http"
"net/http/httptest"
sc "github.com/goharbor/harbor/src/api/scan"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
internal_errors "github.com/goharbor/harbor/src/internal/error"
sc "github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/pkg/scan/report"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
serror "github.com/goharbor/harbor/src/server/error"
"github.com/goharbor/harbor/src/server/middleware"
"github.com/pkg/errors"
"net/http"
"net/http/httptest"
)
// Middleware handle docker pull vulnerable check