Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2025-02-03 13:31:22 +01:00
Code Issues Projects Releases Wiki Activity

improve the scan controlling

Browse Source 
- add LCM control to the robot account generated for scanning
- improve the scan webhook
- remove reprots when related artifact is deleted
- update report manager/scan controller and other components to support above cases
- add artifact manager/comtroller to list artifacts

Signed-off-by: Steven Zou <szou@vmware.com>
This commit is contained in:
Steven Zou 2019-10-21 20:07:00 +08:00
parent be5a265dd2
commit dff1ee07fc
29 changed files with 675 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/core/api/repository.go
View File

@ -19,7 +19,6 @@ import (
"errors"
"fmt"
"io/ioutil"
"net/http"
"sort"
"strconv"
"strings"
@ -258,6 +257,22 @@ func (ra *RepositoryAPI) Delete() {
tags = append(tags, tag)
}
// Retrieve the manifests of the tags first
// If tag not exist, mapping with empty digest
digests := make(map[string]string)
for _, t := range tags {
dig, exists, err := rc.ManifestExist(t)
if err != nil {
log.Errorf("Failed to check the digest of tag: %s:%s, error: %v", repoName, t, err.Error())
ra.SendInternalServerError(err)
return
}
if exists {
digests[t] = dig
}
}
if config.WithNotary() {
signedTags, err := getSignatures(ra.SecurityCtx.GetUsername(), repoName)
if err != nil {
@ -267,14 +282,15 @@ func (ra *RepositoryAPI) Delete() {
}
for _, t := range tags {
digest, _, err := rc.ManifestExist(t)
if err != nil {
log.Errorf("Failed to Check the digest of tag: %s, error: %v", t, err.Error())
ra.SendInternalServerError(err)
return
dig, exists := digests[t]
if !exists {
log.Errorf("No digest found for image: %s:%s, ignore the following signature check", repoName, t)
continue
}
log.Debugf("Tag: %s, digest: %s", t, digest)
if _, ok := signedTags[digest]; ok {
log.Debugf("Tag: %s, digest: %s", t, digests[t])
if _, ok := signedTags[dig]; ok {
log.Errorf("Found signed tag, repository: %s, tag: %s, deletion will be canceled", repoName, t)
ra.SendPreconditionFailedError(fmt.Errorf("tag %s is signed", t))
return
@ -288,12 +304,13 @@ func (ra *RepositoryAPI) Delete() {
ra.SendInternalServerError(fmt.Errorf("failed to delete labels of image %s: %v", image, err))
return
}
if err = rc.DeleteTag(t); err != nil {
if regErr, ok := err.(*commonhttp.Error); ok {
if regErr.Code == http.StatusNotFound {
continue
}
}
if len(digests[t]) == 0 {
log.Errorf("No digest found for image: %s:%s, ignore the following deletion", repoName, t)
continue
}
if err = rc.DeleteManifest(digests[t]); err != nil {
ra.ParseAndHandleError(fmt.Sprintf("failed to delete tag %s", t), err)
return
}
@ -337,6 +354,7 @@ func (ra *RepositoryAPI) Delete() {
imgDelMetadata := &notifierEvt.ImageDelMetaData{
Project: project,
Tags: tags,
Digests: digests,
RepoName: repoName,
OccurAt: time.Now(),
Operator: ra.SecurityCtx.GetUsername(),

4
src/core/api/scan_test.go
View File

@ -211,3 +211,7 @@ func (msc *MockScanAPIController) HandleJobHooks(trackID string, change *job.Sta
return args.Error(0)
}
func (msc *MockScanAPIController) DeleteReports(digests ...string) error {
return nil
}

8
src/core/api/scanners_test.go
View File

@ -370,11 +370,3 @@ func (m *MockScannerAPIController) GetMetadata(registrationUUID string) (*v1.Sca
return sam.(*v1.ScannerAdapterMetadata), nil
}
// IsScannerAvailable ...
// TODO: Remove it when the interface is changed
func (m *MockScannerAPIController) IsScannerAvailable(projectID int64) (bool, error) {
args := m.Called(projectID)
return args.Bool(0), args.Error(1)
}

4
src/core/main.go
View File

@ -23,6 +23,8 @@ import (
"syscall"
"time"
"github.com/goharbor/harbor/src/pkg/scan/event"
"github.com/astaxie/beego"
_ "github.com/astaxie/beego/session/redis"
"github.com/goharbor/harbor/src/common/dao"
@ -241,6 +243,8 @@ func main() {
log.Info("initializing notification...")
notification.Init()
// Initialize the event handlers for handling artifact cascade deletion
event.Init()
filter.Init()
beego.InsertFilter("/*", beego.BeforeRouter, filter.SecurityFilter)

15
src/core/notifier/event/event.go
View File

@ -31,6 +31,7 @@ type Metadata interface {
type ImageDelMetaData struct {
Project *models.Project
Tags []string
Digests map[string]string
OccurAt time.Time
Operator string
RepoName string
@ -46,7 +47,10 @@ func (i *ImageDelMetaData) Resolve(evt *Event) error {
RepoName: i.RepoName,
}
for _, t := range i.Tags {
res := &model.ImgResource{Tag: t}
res := &model.ImgResource{
Tag: t,
Digest: i.Digests[t],
}
data.Resource = append(data.Resource, res)
}
evt.Topic = model.DeleteImageTopic
@ -194,15 +198,18 @@ type ScanImageMetaData struct {
func (si *ScanImageMetaData) Resolve(evt *Event) error {
var eventType string
var topic string
if si.Status == models.JobFinished {
switch si.Status {
case models.JobFinished:
eventType = notifyModel.EventTypeScanningCompleted
topic = model.ScanningCompletedTopic
} else if si.Status == models.JobError {
case models.JobError, models.JobStopped:
eventType = notifyModel.EventTypeScanningFailed
topic = model.ScanningFailedTopic
} else {
default:
return errors.New("not supported scan hook status")
}
data := &model.ScanImageEvent{
EventType: eventType,
Artifact: si.Artifact,

18
src/core/notifier/handler/notification/scan_image_handler.go
View File

@ -1,6 +1,8 @@
package notification
import (
"time"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
@ -99,6 +101,22 @@ func constructScanImagePayload(event *model.ScanImageEvent, project *models.Proj
return nil, errors.Wrap(err, "construct scan payload")
}
// Wait for reasonable time to make sure the report is ready
// Interval=500ms and total time = 5s
// If the report is still not ready in the total time, then failed at then
for i := 0; i < 10; i++ {
// First check in case it is ready
if re, err := scan.DefaultController.GetReport(event.Artifact, []string{v1.MimeTypeNativeReport}); err == nil {
if len(re) > 0 && len(re[0].Report) > 0 {
break
}
} else {
log.Error(errors.Wrap(err, "construct scan payload: wait report ready loop"))
}
time.Sleep(500 * time.Millisecond)
}
// Add scan overview
summaries, err := scan.DefaultController.GetSummary(event.Artifact, []string{v1.MimeTypeNativeReport})
if err != nil {

16
src/core/service/notifications/jobs/handler.go
View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"time"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
@ -27,7 +29,6 @@ import (
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/pkg/retention"
sc "github.com/goharbor/harbor/src/pkg/scan"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/operation/hook"
"github.com/goharbor/harbor/src/replication/policy/scheduler"
@ -93,15 +94,24 @@ func (h *Handler) Prepare() {
// HandleScan handles the webhook of scan job
func (h *Handler) HandleScan() {
log.Debugf("received san job status update event: job UUID: %s, status-%s, track id-%s", h.change.JobID, h.status, h.trackID)
log.Debugf("Received scan job status update event: job UUID: %s, status: %s, track_id: %s, is checkin: %v",
h.change.JobID,
h.status,
h.trackID,
len(h.checkIn) > 0,
)
// Trigger image scan webhook event only for JobFinished and JobError status
if h.status == models.JobFinished || h.status == models.JobError {
if h.status == models.JobFinished ||
h.status == models.JobError ||
h.status == models.JobStopped {
// Get the required info from the job parameters
req, err := sc.ExtractScanReq(h.change.Metadata.Parameters)
if err != nil {
log.Error(errors.Wrap(err, "scan job hook handler: event publish"))
} else {
log.Debugf("Scan %s for artifact: %#v", h.status, req.Artifact)
e := &event.Event{}
metaData := &event.ScanImageMetaData{
Artifact: req.Artifact,

19
src/core/service/notifications/registry/handler.go
View File

@ -33,7 +33,7 @@ import (
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/adapter"
rep_event "github.com/goharbor/harbor/src/replication/event"
repevent "github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/model"
"github.com/pkg/errors"
)
@ -44,7 +44,6 @@ type NotificationHandler struct {
}
const manifestPattern = `^application/vnd.docker.distribution.manifest.v\d\+(json|prettyjws)`
const vicPrefix = "vic/"
// Post handles POST request, and records audit log or refreshes cache based on event.
func (n *NotificationHandler) Post() {
@ -142,8 +141,8 @@ func (n *NotificationHandler) Post() {
// TODO: handle image delete event and chart event
go func() {
e := &rep_event.Event{
Type: rep_event.EventTypeImagePush,
e := &repevent.Event{
Type: repevent.EventTypeImagePush,
Resource: &model.Resource{
Type: model.ResourceTypeImage,
Metadata: &model.ResourceMetadata{
@ -245,7 +244,7 @@ func (n *NotificationHandler) Post() {
}
func filterEvents(notification *models.Notification) ([]*models.Event, error) {
events := []*models.Event{}
events := make([]*models.Event, 0)
for _, event := range notification.Events {
log.Debugf("receive an event: \n----ID: %s \n----target: %s:%s \n----digest: %s \n----action: %s \n----mediatype: %s \n----user-agent: %s", event.ID, event.Target.Repository,
@ -285,13 +284,19 @@ func checkEvent(event *models.Event) bool {
}
func autoScanEnabled(project *models.Project) bool {
available, err := scanner.DefaultController.IsScannerAvailable(project.ProjectID)
r, err := scanner.DefaultController.GetRegistrationByProject(project.ProjectID)
if err != nil {
log.Error(errors.Wrap(err, "check auto scan enable"))
return false
}
return available && project.AutoScan()
// In case
if r == nil {
log.Errorf("no scanner is available for project: %s", project.Name)
return false
}
return !r.Disabled && project.AutoScan()
}
// Render returns nil as it won't render any template.

40
src/pkg/art/basic_controller.go Normal file
View File

@ -0,0 +1,40 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package art
import (
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/q"
)
// DefaultController for easy referring as a singleton instance
var DefaultController = NewController()
// basicController is the default implementation of controller
type basicController struct {
m Manager
}
// NewController
func NewController() Controller {
return &basicController{
m: NewManager(),
}
}
// List artifacts
func (b *basicController) List(query *q.Query) ([]*models.Artifact, error) {
return b.m.List(query)
}

87
src/pkg/art/basic_controller_test.go Normal file
View File

@ -0,0 +1,87 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package art
import (
"testing"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
// TestControllerSuite is a test suite for testing controller.
type TestControllerSuite struct {
suite.Suite
c *basicController
m *MockManager
}
// TestController is the entry point of TestControllerSuite.
func TestController(t *testing.T) {
suite.Run(t, &TestControllerSuite{})
}
// SetupSuite prepares env for test suite.
func (suite *TestControllerSuite) SetupSuite() {
suite.m = &MockManager{}
suite.c = &basicController{
m: suite.m,
}
}
// TestControllerList ...
func (suite *TestControllerSuite) TestControllerList() {
kws := make(map[string]interface{})
kws["digest"] = "digest-code"
query := &q.Query{
Keywords: kws,
}
artifacts := []*models.Artifact{
{
ID: 1000,
PID: 1,
Repo: "library/busybox",
Tag: "dev",
Digest: "digest-code",
Kind: "image",
},
}
suite.m.On("List", query).Return(artifacts, nil)
arts, err := suite.c.List(query)
require.NoError(suite.T(), err)
suite.Equal(1, len(arts))
}
// MockManager ...
type MockManager struct {
mock.Mock
}
// List ...
func (mm *MockManager) List(query *q.Query) ([]*models.Artifact, error) {
args := mm.Called(query)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).([]*models.Artifact), args.Error(1)
}

72
src/pkg/art/basic_manager.go Normal file
View File

@ -0,0 +1,72 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package art
import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/pkg/errors"
)
// basicManager is the default implementation of artifact manager
type basicManager struct{}
// NewManager creates a new basic manager as the default one.
func NewManager() Manager {
return &basicManager{}
}
// List artifacts
func (b *basicManager) List(query *q.Query) ([]*models.Artifact, error) {
aq := &models.ArtifactQuery{}
makeArtQuery(aq, query)
l, err := dao.ListArtifacts(aq)
if err != nil {
return nil, errors.Wrap(err, "artifact manager: list")
}
return l, nil
}
func makeArtQuery(aq *models.ArtifactQuery, query *q.Query) {
if aq == nil {
return // do nothing
}
if query != nil {
if len(query.Keywords) > 0 {
for k, v := range query.Keywords {
switch k {
case "project_id":
aq.PID = v.(int64)
case "repo":
aq.Repo = v.(string)
case "tag":
aq.Tag = v.(string)
case "digest":
aq.Digest = v.(string)
default:
}
}
}
if query.PageNumber > 0 && query.PageSize > 0 {
aq.Page = query.PageNumber
aq.Size = query.PageSize
}
}
}

56
src/pkg/art/basic_manager_test.go Normal file
View File

@ -0,0 +1,56 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package art
import (
"testing"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
// TestManagerSuite is a test suite for testing manager
type TestManagerSuite struct {
suite.Suite
m *basicManager
}
// TestManager is the entry point of TestManagerSuite
func TestManager(t *testing.T) {
suite.Run(t, &TestManagerSuite{})
}
// SetupSuite prepares env for test suite
func (suite *TestManagerSuite) SetupSuite() {
dao.PrepareTestForPostgresSQL()
suite.m = &basicManager{}
}
// TestManagerSuiteList ...
func (suite *TestManagerSuite) TestManagerSuiteList() {
kws := make(map[string]interface{})
kws["digest"] = "fake-digest"
l, err := suite.m.List(&q.Query{
Keywords: kws,
})
require.NoError(suite.T(), err)
suite.Equal(0, len(l))
}

33
src/pkg/art/controller.go Normal file
View File

@ -0,0 +1,33 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package art
import (
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/q"
)
// Controller for artifact
type Controller interface {
// List the artifacts with queries
//
// Arguments:
// query *q.Query : optional query parameters
//
// Returns:
// []*models.Artifact : the queried artifacts
// error : non nil error if any errors occurred
List(query *q.Query) ([]*models.Artifact, error)
}

32
src/pkg/art/manager.go Normal file
View File

@ -0,0 +1,32 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package art
import (
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/q"
)
type Manager interface {
// List the artifacts with queries
//
// Arguments:
// query *q.Query : optional query parameters
//
// Returns:
// []*models.Artifact : the queried artifacts
// error : non nil error if any errors occurred
List(query *q.Query) ([]*models.Artifact, error)
}

34
src/pkg/scan/api/scan/base_controller.go
View File

@ -19,6 +19,8 @@ import (
"fmt"
"time"
"github.com/goharbor/harbor/src/common/utils/log"
cj "github.com/goharbor/harbor/src/common/job"
jm "github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/common/rbac"
@ -294,6 +296,22 @@ func (bc *basicController) HandleJobHooks(trackID string, change *job.StatusChan
return errors.New("nil change object")
}
// Clear robot account
// All final statuses (success, error and stopped) share the same code.
// Only need to check one of them.
if job.Status(change.Status).Compare(job.SuccessStatus) == 0 {
if v, ok := change.Metadata.Parameters[sca.JobParameterRobotID]; ok {
if rid, y := v.(float64); y {
if err := robot.RobotCtr.DeleteRobotAccount(int64(rid)); err != nil {
// Should not block the main flow, just logged
log.Error(errors.Wrap(err, "scan controller: handle job hook"))
} else {
log.Debugf("Robot account with id %d for the scan %s is removed", rid, trackID)
}
}
}
}
// Check in data
if len(change.CheckIn) > 0 {
checkInReport := &sca.CheckInReport{}
@ -326,12 +344,17 @@ func (bc *basicController) HandleJobHooks(trackID string, change *job.StatusChan
return bc.manager.UpdateStatus(trackID, change.Status, change.Metadata.Revision)
}
// DeleteReports ...
func (bc *basicController) DeleteReports(digests ...string) error {
return bc.manager.DeleteByDigests(digests...)
}
// makeAuthorization creates authorization from a robot account based on the arguments for scanning.
func (bc *basicController) makeAuthorization(pid int64, repository string, ttl int64) (string, error) {
func (bc *basicController) makeAuthorization(pid int64, repository string, ttl int64) (string, int64, error) {
// Use uuid as name to avoid duplicated entries.
UUID, err := bc.uuid()
if err != nil {
return "", errors.Wrap(err, "scan controller: make robot account")
return "", -1, errors.Wrap(err, "scan controller: make robot account")
}
expireAt := time.Now().UTC().Add(time.Duration(ttl) * time.Second).Unix()
@ -353,13 +376,13 @@ func (bc *basicController) makeAuthorization(pid int64, repository string, ttl i
rb, err := bc.rc.CreateRobotAccount(robotReq)
if err != nil {
return "", errors.Wrap(err, "scan controller: make robot account")
return "", -1, errors.Wrap(err, "scan controller: make robot account")
}
basic := fmt.Sprintf("%s:%s", rb.Name, rb.Token)
encoded := base64.StdEncoding.EncodeToString([]byte(basic))
return fmt.Sprintf("Basic %s", encoded), nil
return fmt.Sprintf("Basic %s", encoded), rb.ID, nil
}
// launchScanJob launches a job to run scan
@ -370,7 +393,7 @@ func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact,
}
// Make authorization from a robot account with 30 minutes
authorization, err := bc.makeAuthorization(artifact.NamespaceID, artifact.Repository, 1800)
authorization, rID, err := bc.makeAuthorization(artifact.NamespaceID, artifact.Repository, 1800)
if err != nil {
return "", errors.Wrap(err, "scan controller: launch scan job")
}
@ -398,6 +421,7 @@ func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact,
params[sca.JobParamRegistration] = rJSON
params[sca.JobParameterRequest] = sJSON
params[sca.JobParameterMimes] = mimes
params[sca.JobParameterRobotID] = rID
// Launch job
callbackURL, err := bc.config(configCoreInternalAddr)

6
src/pkg/scan/api/scan/base_controller_test.go
View File

@ -345,6 +345,12 @@ func (mrm *MockReportManager) Get(uuid string) (*scan.Report, error) {
return args.Get(0).(*scan.Report), args.Error(1)
}
func (mrm *MockReportManager) DeleteByDigests(digests ...string) error {
args := mrm.Called(digests)
return args.Error(0)
}
// MockScannerController ...
type MockScannerController struct {
mock.Mock

9
src/pkg/scan/api/scan/controller.go
View File

@ -77,4 +77,13 @@ type Controller interface {
// Returns:
// error : non nil error if any errors occurred
HandleJobHooks(trackID string, change *job.StatusChange) error
// Delete the reports related with the specified digests
//
// Arguments:
// digests ...string : specify one or more digests whose reports will be deleted
//
// Returns:
// error : non nil error if any errors occurred
DeleteReports(digests ...string) error
}

63
src/pkg/scan/api/scanner/base_controller.go
View File

@ -57,17 +57,16 @@ func (bc *basicController) ListRegistrations(query *q.Query) ([]*scanner.Registr
return nil, errors.Wrap(err, "api controller: list registrations")
}
for _, r := range l {
_, err = bc.Ping(r)
r.Health = err == nil
}
return l, nil
}
// CreateRegistration ...
func (bc *basicController) CreateRegistration(registration *scanner.Registration) (string, error) {
// TODO: Check connection of the registration.
// Check if the registration is available
if _, err := bc.Ping(registration); err != nil {
return "", errors.Wrap(err, "api controller: create registration")
}
// Check if there are any registrations already existing.
l, err := bc.manager.List(&q.Query{
PageSize: 1,
@ -89,12 +88,9 @@ func (bc *basicController) CreateRegistration(registration *scanner.Registration
func (bc *basicController) GetRegistration(registrationUUID string) (*scanner.Registration, error) {
r, err := bc.manager.Get(registrationUUID)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "api controller: get registration")
}
_, err = bc.Ping(r)
r.Health = err == nil
return r, nil
}
@ -214,18 +210,6 @@ func (bc *basicController) GetRegistrationByProject(projectID int64) (*scanner.R
}
}
// Check status by the client later
if registration != nil {
if meta, err := bc.Ping(registration); err == nil {
registration.Scanner = meta.Scanner.Name
registration.Vendor = meta.Scanner.Vendor
registration.Version = meta.Scanner.Version
registration.Health = true
} else {
registration.Health = false
}
}
return registration, err
}
@ -300,38 +284,3 @@ func (bc *basicController) GetMetadata(registrationUUID string) (*v1.ScannerAdap
return bc.Ping(r)
}
// IsScannerAvailable ...
// TODO: This method will be removed if we change the method of getting project
// registration without ping later.
func (bc *basicController) IsScannerAvailable(projectID int64) (bool, error) {
if projectID == 0 {
return false, errors.New("invalid project ID")
}
// First, get it from the project metadata
m, err := bc.proMetaMgr.Get(projectID, proScannerMetaKey)
if err != nil {
return false, errors.Wrap(err, "api controller: check scanner availability")
}
var registration *scanner.Registration
if len(m) > 0 {
if registrationID, ok := m[proScannerMetaKey]; ok && len(registrationID) > 0 {
registration, err = bc.manager.Get(registrationID)
if err != nil {
return false, errors.Wrap(err, "api controller: check scanner availability")
}
}
}
if registration == nil {
// Second, get the default one
registration, err = bc.manager.GetDefault()
if err != nil {
return false, errors.Wrap(err, "api controller: check scanner availability")
}
}
return registration != nil && !registration.Disabled, nil
}

10
src/pkg/scan/api/scanner/controller.go
View File

@ -135,14 +135,4 @@ type Controller interface {
// *v1.ScannerAdapterMetadata : metadata returned by the scanner if successfully ping
// error : non nil error if any errors occurred
GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error)
// IsScannerAvailable checks if the scanner is available for the specified project.
//
// Arguments:
// projectID int64 : the ID of the given project
//
// Returns:
// bool : the scanner if configured for the specified project
// error : non nil error if any errors occurred
IsScannerAvailable(projectID int64) (bool, error)
}

9
src/pkg/scan/dao/scanner/model.go
View File

@ -37,21 +37,16 @@ type Registration struct {
URL string `orm:"column(url);unique;size(512)" json:"url"`
Disabled bool `orm:"column(disabled);default(true)" json:"disabled"`
IsDefault bool `orm:"column(is_default);default(false)" json:"is_default"`
Health bool `orm:"-" json:"health"`
Health bool `orm:"-" json:"-"` // Reserved for future use
// Authentication settings
// "None","Basic" and "Bearer" can be supported
// "","Basic", "Bearer" and api key header "X-ScannerAdapter-API-Key" can be supported
Auth string `orm:"column(auth);size(16)" json:"auth"`
AccessCredential string `orm:"column(access_cred);null;size(512)" json:"access_credential,omitempty"`
// Http connection settings
SkipCertVerify bool `orm:"column(skip_cert_verify);default(false)" json:"skip_certVerify"`
// Extra info about the scanner
Scanner string `orm:"-" json:"scanner,omitempty"`
Vendor string `orm:"-" json:"vendor,omitempty"`
Version string `orm:"-" json:"version,omitempty"`
// Timestamps
CreateTime time.Time `orm:"column(create_time);auto_now_add;type(datetime)" json:"create_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now;type(datetime)" json:"update_time"`

30
src/pkg/scan/event/init.go Normal file
View File

@ -0,0 +1,30 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package event
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/notifier"
"github.com/goharbor/harbor/src/core/notifier/model"
"github.com/pkg/errors"
)
// Init the events for scan
func Init() {
log.Debugf("Subscribe topic %s for cascade deletion of scan reports")
err := notifier.Subscribe(model.DeleteImageTopic, NewOnDelImageHandler())
log.Error(errors.Wrap(err, "register on delete image handler: init: scan"))
}

97
src/pkg/scan/event/notification.go Normal file
View File

@ -0,0 +1,97 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package event
import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/notifier"
"github.com/goharbor/harbor/src/core/notifier/model"
"github.com/goharbor/harbor/src/pkg/art"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/pkg/errors"
)
// scanCtlGetter for getting a scan controller reference to avoid package importing order issue.
type scanCtlGetter func() scan.Controller
// artCtlGetter for getting a artifact controller reference to avoid package importing order issue.
type artCtlGetter func() art.Controller
// onDelImageHandler is a handler to listen to the internal delete image event.
type onDelImageHandler struct {
// scan controller
scanCtl scanCtlGetter
// artifact controller
artCtl artCtlGetter
}
// NewOnDelImageHandler creates a new handler to handle on del event.
func NewOnDelImageHandler() notifier.NotificationHandler {
return &onDelImageHandler{
scanCtl: func() scan.Controller {
return scan.DefaultController
},
artCtl: func() art.Controller {
return art.DefaultController
},
}
}
func (o *onDelImageHandler) Handle(value interface{}) error {
if value == nil {
return errors.New("delete image event handler: nil value ")
}
evt, ok := value.(*model.ImageEvent)
if !ok {
return errors.New("delete image event handler: malformed image event model")
}
log.Debugf("clear the scan reports as receiving event %s", evt.EventType)
digests := make([]string, 0)
query := &q.Query{
Keywords: make(map[string]interface{}),
}
for _, res := range evt.Resource {
// Check if it is safe to delete the reports.
query.Keywords["digest"] = res.Digest
l, err := o.artCtl().List(query)
if err != nil {
// Just logged
log.Error(errors.Wrap(err, "delete image event handler"))
// Passed for safe consideration
continue
}
if len(l) == 0 {
digests = append(digests, res.Digest)
log.Debugf("prepare to remove the scan report linked with artifact: %s", res.Digest)
}
}
if err := o.scanCtl().DeleteReports(digests...); err != nil {
return errors.Wrap(err, "delete image event handler")
}
return nil
}
func (o *onDelImageHandler) IsStateful() bool {
return false
}

8
src/pkg/scan/job.go
View File

@ -37,6 +37,8 @@ const (
JobParameterRequest = "scanRequest"
// JobParameterMimes ...
JobParameterMimes = "mimeTypes"
// JobParameterRobotID ...
JobParameterRobotID = "robotID"
checkTimeout = 30 * time.Minute
firstCheckInterval = 2 * time.Second
@ -101,6 +103,10 @@ func (j *Job) Validate(params job.Parameters) error {
return errors.Wrap(err, "job validate")
}
// No need to check param robotID which os treated as an optional one.
// It is used to clear the generated robot account to reduce dirty data.
// Failure of doing this will not influence the main flow.
return nil
}
@ -125,6 +131,8 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error {
return logAndWrapError(myLogger, err, "scan job: get client")
}
// Ignore the namespace ID here
req.Artifact.NamespaceID = 0
resp, err := client.SubmitScan(req)
if err != nil {
return logAndWrapError(myLogger, err, "scan job: submit scan request")

32
src/pkg/scan/report/base_manager.go
View File

@ -192,3 +192,35 @@ func (bm *basicManager) UpdateReportData(uuid string, report string, rev int64)
return scan.UpdateReportData(uuid, report, rev)
}
// DeleteByDigests ...
func (bm *basicManager) DeleteByDigests(digests ...string) error {
if len(digests) == 0 {
// Nothing to do
return nil
}
kws := make(map[string]interface{})
kws["digest"] = digests
query := &q.Query{
Keywords: kws,
}
rs, err := scan.ListReports(query)
if err != nil {
return errors.Wrap(err, "report manager: delete by digests")
}
// Return the combined errors at last
for _, r := range rs {
if er := scan.DeleteReport(r.UUID); er != nil {
if err == nil {
err = er
} else {
err = errors.Wrap(er, err.Error())
}
}
}
return err
}

22
src/pkg/scan/report/base_manager_test.go
View File

@ -166,3 +166,25 @@ func (suite *TestManagerSuite) TestManagerUpdateReportData() {
assert.Equal(suite.T(), "{\"a\":1000}", l[0].Report)
}
// TestManagerDeleteByDigests ...
func (suite *TestManagerSuite) TestManagerDeleteByDigests() {
// Mock new data
rp := &scan.Report{
Digest: "d2000",
RegistrationUUID: "ruuid",
MimeType: v1.MimeTypeNativeReport,
TrackID: "tid002",
}
uuid, err := suite.m.Create(rp)
require.NoError(suite.T(), err)
require.NotEmpty(suite.T(), uuid)
err = suite.m.DeleteByDigests("d2000")
require.NoError(suite.T(), err)
r, err := suite.m.Get(uuid)
suite.NoError(err)
suite.Nil(r)
}

11
src/pkg/scan/report/manager.go
View File

@ -81,10 +81,19 @@ type Manager interface {
// Get the report for the given uuid.
//
// Arguments:
// uuid string : uuid of the scan report
// uuid string : uuid of the scan report
//
// Returns:
// *scan.Report : scan report
// error : non nil error if any errors occurred
Get(uuid string) (*scan.Report, error)
// Delete the reports related with the specified digests (one or more...)
//
// Arguments:
// digests ...string : specify one or more digests whose reports will be deleted
//
// Returns:
// error : non nil error if any errors occurred
DeleteByDigests(digests ...string) error
}

7
src/pkg/scan/report/summary.go
View File

@ -81,7 +81,7 @@ func GenerateNativeSummary(r *scan.Report, options ...Option) (interface{}, erro
sum.ReportID = r.UUID
sum.StartTime = r.StartTime
sum.EndTime = r.EndTime
sum.Duration = r.EndTime.Unix() - r.EndTime.Unix()
sum.Duration = r.EndTime.Unix() - r.StartTime.Unix()
if len(ops.CVEWhitelist) > 0 {
sum.CVEBypassed = make([]string, 0)
}
@ -140,6 +140,11 @@ func GenerateNativeSummary(r *scan.Report, options ...Option) (interface{}, erro
if v.Severity.Code() > overallSev.Code() {
overallSev = v.Severity
}
// If the CVE item has a fixable version
if len(v.FixVersion) > 0 {
vsum.Fixable++
}
}
sum.Summary = vsum

6
src/pkg/scan/rest/v1/models.go
View File

@ -68,7 +68,7 @@ type ScannerAdapterMetadata struct {
// Artifact represents an artifact stored in Registry.
type Artifact struct {
// ID of the namespace (project). It will not be sent to scanner adapter.
NamespaceID int64 `json:"-"`
NamespaceID int64 `json:"namespace_id,omitempty"`
// The full name of a Harbor repository containing the artifact, including the namespace.
// For example, `library/oracle/nosql`.
Repository string `json:"repository"`
@ -87,8 +87,8 @@ type Artifact struct {
type Registry struct {
// A base URL of the Docker Registry v2 API exposed by Harbor.
URL string `json:"url"`
// An optional value of the HTTP Authorization header sent with each request to the Docker Registry v2 API.
// For example, `Bearer: JWTTOKENGOESHERE`.
// An optional value of the HTTP Authorization header sent with each request to the Docker Registry for getting or exchanging token.
// For example, `Basic: Base64(username:password)`.
Authorization string `json:"authorization"`
}

1
src/pkg/scan/vuln/summary.go
View File

@ -35,6 +35,7 @@ type NativeReportSummary struct {
// and numbers of each severity level.
type VulnerabilitySummary struct {
Total int `json:"total"`
Fixable int `json:"fixable"`
Summary SeveritySummary `json:"summary"`
}