Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-21 17:55:30 +01:00
Code Issues Projects Releases Wiki Activity

Store vulnerability data from scanner into a relational format (#13616)

Browse Source 
feat: Store vulnerability report from scanner into a relational format

Convert vulnerability report JSON obtained  from scanner into a relational format describe in:https://github.com/goharbor/community/pull/145

Signed-off-by: prahaladdarkin <prahaladd@vmware.com>
This commit is contained in:
prahaladdarkin 2020-12-25 06:17:46 +05:30 committed by GitHub
parent 47841a04b9
commit a890b28e1e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 1537 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,5 +1,6 @@
!/contrib/helm/harbor
!/contrib/helm/harbor
*.code-workspace
make/harbor.yml
make/docker-compose.yml
make/common/config/*

41
make/migrations/postgresql/0050_2.2.0_schema.up.sql
View File

@ -472,3 +472,44 @@ BEGIN
update properties set v = cast(duration_in_days as text) WHERE k = 'robot_token_duration';
END IF;
END $$;
/*
Common vulnerability reporting schema.
Github proposal link : https://github.com/goharbor/community/pull/145
*/
-- --------------------------------------------------
-- Table Structure for `main.VulnerabilityRecord`
-- --------------------------------------------------
CREATE TABLE IF NOT EXISTS "vulnerability_record" (
"id" serial NOT NULL PRIMARY KEY,
"cve_id" text NOT NULL DEFAULT '' ,
"registration_uuid" text NOT NULL DEFAULT '',
"package" text NOT NULL DEFAULT '' ,
"package_version" text NOT NULL DEFAULT '' ,
"package_type" text NOT NULL DEFAULT '' ,
"severity" text NOT NULL DEFAULT '' ,
"fixed_version" text,
"urls" text,
"cvss_score_v3" double precision,
"cvss_score_v2" double precision,
"cvss_vector_v3" text,
"cvss_vector_v2" text,
"description" text,
"cwe_ids" text,
"vendor_attributes" json,
UNIQUE ("cve_id", "registration_uuid", "package", "package_version"),
CONSTRAINT fk_registration_uuid FOREIGN KEY(registration_uuid) REFERENCES scanner_registration(uuid) ON DELETE CASCADE
);
-- --------------------------------------------------
-- Table Structure for `main.ReportVulnerabilityRecord`
-- --------------------------------------------------
CREATE TABLE IF NOT EXISTS "report_vulnerability_record" (
"id" serial NOT NULL PRIMARY KEY,
"report_uuid" text NOT NULL DEFAULT '' ,
"vuln_record_id" bigint NOT NULL DEFAULT 0 ,
UNIQUE ("report_uuid", "vuln_record_id"),
CONSTRAINT fk_vuln_record_id FOREIGN KEY(vuln_record_id) REFERENCES vulnerability_record(id) ON DELETE CASCADE,
CONSTRAINT fk_report_uuid FOREIGN KEY(report_uuid) REFERENCES scan_report(uuid) ON DELETE CASCADE
);

4
src/controller/event/handler/webhook/scan/scan.go
View File

@ -131,7 +131,7 @@ func constructScanImagePayload(event *event.ScanImageEvent, project *models.Proj
// If the report is still not ready in the total time, then failed at then
for i := 0; i < 10; i++ {
// First check in case it is ready
if re, err := scan.DefaultController.GetReport(ctx, art, []string{v1.MimeTypeNativeReport}); err == nil {
if re, err := scan.DefaultController.GetReport(ctx, art, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport}); err == nil {
if len(re) > 0 && len(re[0].Report) > 0 {
break
}
@ -143,7 +143,7 @@ func constructScanImagePayload(event *event.ScanImageEvent, project *models.Proj
}
// Add scan overview
summaries, err := scan.DefaultController.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport})
summaries, err := scan.DefaultController.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport})
if err != nil {
return nil, errors.Wrap(err, "construct scan payload")
}

12
src/controller/p2p/preheat/enforcer.go
View File

@ -477,7 +477,7 @@ func (de *defaultEnforcer) startTask(ctx context.Context, executionID int64, can
// getVulnerabilitySev gets the severity code value for the given artifact with allowlist option set
func (de *defaultEnforcer) getVulnerabilitySev(ctx context.Context, p *models.Project, art *artifact.Artifact) (uint, error) {
al := p.CVEAllowlist.CVESet()
r, err := de.scanCtl.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport}, report.WithCVEAllowlist(&al))
r, err := de.scanCtl.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport}, report.WithCVEAllowlist(&al))
if err != nil {
if errors.IsNotFoundErr(err) {
// no vulnerability report
@ -487,11 +487,17 @@ func (de *defaultEnforcer) getVulnerabilitySev(ctx context.Context, p *models.Pr
return defaultSeverityCode, errors.Wrap(err, "get vulnerability severity")
}
// Severity is based on the native report format.
// Severity is based on the native report format or the generic vulnerability report format.
// In case no supported report format, treat as same to the no report scenario
sum, ok := r[v1.MimeTypeNativeReport]
if !ok {
return defaultSeverityCode, nil
// check if a report with MimeTypeGenericVulnerabilityReport is present.
// return the default severity code only if it does not exist
sum, ok = r[v1.MimeTypeGenericVulnerabilityReport]
if !ok {
return defaultSeverityCode, nil
}
}
sm, ok := sum.(*vuln.NativeReportSummary)

5
src/controller/p2p/preheat/enforcer_test.go
View File

@ -106,7 +106,7 @@ func (suite *EnforcerTestSuite) SetupSuite() {
fakeScanCtl.On("GetSummary",
context.TODO(),
mock.AnythingOfType("*artifact.Artifact"),
[]string{v1.MimeTypeNativeReport},
[]string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport},
mock.AnythingOfType("report.Option"),
).Return(mockVulnerabilitySummary(), nil)
@ -310,5 +310,8 @@ func mockVulnerabilitySummary() map[string]interface{} {
v1.MimeTypeNativeReport: &vuln.NativeReportSummary{
Severity: vuln.Low,
},
v1.MimeTypeGenericVulnerabilityReport: &vuln.NativeReportSummary{
Severity: vuln.Low,
},
}
}

28
src/controller/scan/base_controller.go
View File

@ -37,6 +37,7 @@ import (
sca "github.com/goharbor/harbor/src/pkg/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/postprocessors"
"github.com/goharbor/harbor/src/pkg/scan/report"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
@ -86,6 +87,8 @@ type basicController struct {
execMgr task.ExecutionManager
taskMgr task.Manager
// Converter for V1 report to V2 report
reportConverter postprocessors.NativeScanReportConverter
}
// NewController news a scan API controller
@ -125,6 +128,8 @@ func NewController() Controller {
execMgr: task.ExecMgr,
taskMgr: task.Mgr,
// Get the scan V1 to V2 report converters
reportConverter: postprocessors.NewNativeToRelationalSchemaConverter(),
}
}
@ -397,8 +402,8 @@ func (bc *basicController) GetReport(ctx context.Context, artifact *ar.Artifact,
mimes := make([]string, 0)
mimes = append(mimes, mimeTypes...)
if len(mimes) == 0 {
// Retrieve native as default
mimes = append(mimes, v1.MimeTypeNativeReport)
// Retrieve native and the new generic format as default
mimes = append(mimes, v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport)
}
// Get current scanner settings
@ -593,10 +598,19 @@ func (bc *basicController) UpdateReport(ctx context.Context, report *sca.CheckIn
return errors.New("no report found to update data")
}
if err := bc.manager.UpdateReportData(ctx, rpl[0].UUID, report.RawReport); err != nil {
log.Infof("Converting report ID %s to the new V2 schema", rpl[0].UUID)
_, reportData, err := bc.reportConverter.ToRelationalSchema(ctx, rpl[0].UUID, rpl[0].RegistrationUUID, rpl[0].Digest, report.RawReport)
if err != nil {
return errors.Wrapf(err, "Failed to convert vulnerability data to new schema for report UUID : %s", rpl[0].UUID)
}
// update the original report with the new summarized report with all vulnerability data removed.
// this is required since the top level layers relay on the vuln.Report struct that
// contains additional metadata within the report which if stored in the new columns within the scan_report table
// would be redundant
if err := bc.manager.UpdateReportData(ctx, rpl[0].UUID, reportData); err != nil {
return errors.Wrap(err, "scan controller: handle job hook")
}
log.Infof("Converted report ID %s to the new V2 schema", rpl[0].UUID)
return nil
}
@ -605,7 +619,6 @@ func (bc *basicController) DeleteReports(ctx context.Context, digests ...string)
if err := bc.manager.DeleteByDigests(ctx, digests...); err != nil {
return errors.Wrap(err, "scan controller: delete reports")
}
return nil
}
@ -835,6 +848,11 @@ func (bc *basicController) assembleReports(ctx context.Context, reports ...*scan
} else {
report.Status = job.ErrorStatus.String()
}
completeReport, err := bc.reportConverter.FromRelationalSchema(ctx, report.UUID, report.Digest, report.Report)
if err != nil {
return err
}
report.Report = completeReport
}
return nil

17
src/controller/scan/base_controller_test.go
View File

@ -42,6 +42,7 @@ import (
scannertesting "github.com/goharbor/harbor/src/testing/controller/scanner"
ormtesting "github.com/goharbor/harbor/src/testing/lib/orm"
"github.com/goharbor/harbor/src/testing/mock"
postprocessorstesting "github.com/goharbor/harbor/src/testing/pkg/scan/postprocessors"
reporttesting "github.com/goharbor/harbor/src/testing/pkg/scan/report"
tasktesting "github.com/goharbor/harbor/src/testing/pkg/task"
"github.com/stretchr/testify/assert"
@ -60,11 +61,12 @@ type ControllerTestSuite struct {
artifact *artifact.Artifact
rawReport string
execMgr *tasktesting.ExecutionManager
taskMgr *tasktesting.Manager
reportMgr *reporttesting.Manager
ar artifact.Controller
c Controller
execMgr *tasktesting.ExecutionManager
taskMgr *tasktesting.Manager
reportMgr *reporttesting.Manager
ar artifact.Controller
c Controller
reportConverter *postprocessorstesting.ScanReportV1ToV2Converter
}
// TestController is the entry point of ControllerTestSuite.
@ -277,8 +279,9 @@ func (suite *ControllerTestSuite) SetupSuite() {
cloneCtx: func(ctx context.Context) context.Context { return ctx },
makeCtx: func() context.Context { return context.TODO() },
execMgr: suite.execMgr,
taskMgr: suite.taskMgr,
execMgr: suite.execMgr,
taskMgr: suite.taskMgr,
reportConverter: &postprocessorstesting.ScanReportV1ToV2Converter{},
}
}

16
src/controller/scan/callback_test.go
View File

@ -18,8 +18,6 @@ import (
"context"
"encoding/json"
"fmt"
"testing"
"github.com/goharbor/harbor/src/controller/artifact"
"github.com/goharbor/harbor/src/controller/robot"
"github.com/goharbor/harbor/src/jobservice/job"
@ -30,9 +28,11 @@ import (
artifacttesting "github.com/goharbor/harbor/src/testing/controller/artifact"
robottesting "github.com/goharbor/harbor/src/testing/controller/robot"
"github.com/goharbor/harbor/src/testing/mock"
postprocessorstesting "github.com/goharbor/harbor/src/testing/pkg/scan/postprocessors"
reporttesting "github.com/goharbor/harbor/src/testing/pkg/scan/report"
tasktesting "github.com/goharbor/harbor/src/testing/pkg/task"
"github.com/stretchr/testify/suite"
"testing"
)
type CallbackTestSuite struct {
@ -53,6 +53,7 @@ type CallbackTestSuite struct {
taskMgr *tasktesting.Manager
originalTaskMgr task.Manager
reportConverter *postprocessorstesting.ScanReportV1ToV2Converter
}
func (suite *CallbackTestSuite) SetupSuite() {
@ -73,11 +74,14 @@ func (suite *CallbackTestSuite) SetupSuite() {
task.Mgr = suite.taskMgr
suite.originalScanCtl = DefaultController
suite.reportConverter = &postprocessorstesting.ScanReportV1ToV2Converter{}
suite.scanCtl = &basicController{
makeCtx: context.TODO,
manager: suite.reportMgr,
execMgr: suite.execMgr,
taskMgr: suite.taskMgr,
makeCtx: context.TODO,
manager: suite.reportMgr,
execMgr: suite.execMgr,
taskMgr: suite.taskMgr,
reportConverter: suite.reportConverter,
}
DefaultController = suite.scanCtl
}

6
src/controller/scanner/base_controller.go
View File

@ -296,17 +296,17 @@ func (bc *basicController) Ping(registration *scanner.Registration) (*v1.Scanner
return nil, errors.Errorf("missing %s in consumes_mime_types", v1.MimeTypeDockerArtifact)
}
// v1.MimeTypeNativeReport is required
// either of v1.MimeTypeNativeReport OR v1.MimeTypeGenericVulnerabilityReport is required
found = false
for _, pm := range ca.ProducesMimeTypes {
if pm == v1.MimeTypeNativeReport {
if pm == v1.MimeTypeNativeReport || pm == v1.MimeTypeGenericVulnerabilityReport {
found = true
break
}
}
if !found {
return nil, errors.Errorf("missing %s in produces_mime_types", v1.MimeTypeNativeReport)
return nil, errors.Errorf("missing %s or %s in produces_mime_types", v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport)
}
}

38
src/controller/scanner/base_controller_test.go
View File

@ -66,6 +66,7 @@ func (suite *ControllerTestSuite) SetupTest() {
ProducesMimeTypes: []string{
v1.MimeTypeNativeReport,
v1.MimeTypeRawReport,
v1.MimeTypeGenericVulnerabilityReport,
},
}},
Properties: v1.ScannerProperties{
@ -260,6 +261,43 @@ func (suite *ControllerTestSuite) TestPing() {
suite.NotNil(meta)
}
// TestPingWithGenericMimeType tests ping for scanners supporting MIME type MimeTypeGenericVulnerabilityReport
func (suite *ControllerTestSuite) TestPingWithGenericMimeType() {
m := &v1.ScannerAdapterMetadata{
Scanner: &v1.Scanner{
Name: "Trivy",
Vendor: "Harbor",
Version: "0.1.0",
},
Capabilities: []*v1.ScannerCapability{{
ConsumesMimeTypes: []string{
v1.MimeTypeOCIArtifact,
v1.MimeTypeDockerArtifact,
},
ProducesMimeTypes: []string{
v1.MimeTypeGenericVulnerabilityReport,
v1.MimeTypeRawReport,
},
}},
Properties: v1.ScannerProperties{
"extra": "testing",
},
}
mc := &v1testing.Client{}
mc.On("GetMetadata").Return(m, nil)
mcp := &v1testing.ClientPool{}
mocktesting.OnAnything(mcp, "Get").Return(mc, nil)
suite.c = &basicController{
manager: suite.mMgr,
proMetaMgr: suite.mMeta,
clientPool: mcp,
}
meta, err := suite.c.Ping(suite.sample)
require.NoError(suite.T(), err)
suite.NotNil(meta)
}
// TestGetMetadata ...
func (suite *ControllerTestSuite) TestGetMetadata() {
suite.sample.UUID = "uuid"

63
src/pkg/scan/dao/scan/model.go
View File

@ -43,3 +43,66 @@ func (r *Report) TableUnique() [][]string {
{"digest", "registration_uuid", "mime_type"},
}
}
// VulnerabilityRecord of an individual vulnerability. Identifies an individual vulnerability item in the scan.
// Since multiple scanners could be registered with the projects, each scanner
// would have it's own definition for the same CVE ID. Hence a CVE ID is qualified along
// with the ID of the scanner that owns the CVE record definition.
// The scanner ID would be the same as the RegistrationUUID field of Report.
// Identified by the `cve_id` and `registration_uuid`.
// Relates to the image using the `digest` and to the report using the `report UUID` field
type VulnerabilityRecord struct {
ID int64 `orm:"pk;auto;column(id)"`
CVEID string `orm:"column(cve_id)"`
RegistrationUUID string `orm:"column(registration_uuid)"`
Package string `orm:"column(package)"`
PackageVersion string `orm:"column(package_version)"`
PackageType string `orm:"column(package_type)"`
Severity string `orm:"column(severity)"`
Fix string `orm:"column(fixed_version);null"`
URLs string `orm:"column(urls);null"`
CVE3Score *float64 `orm:"column(cvss_score_v3);null"`
CVE2Score *float64 `orm:"column(cvss_score_v2);null"`
CVSS3Vector string `orm:"column(cvss_vector_v3);null"` // e.g. CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N
CVSS2Vector string `orm:"column(cvss_vector_v2);null"` // e.g. AV:L/AC:M/Au:N/C:P/I:N/A:N
Description string `orm:"column(description);null"`
CWEIDs string `orm:"column(cwe_ids);null"` // e.g. CWE-476,CWE-123,CWE-234
VendorAttributes string `orm:"column(vendor_attributes);type(json);null"`
}
// ReportVulnerabilityRecord is relation table required to optimize data storage for both the
// vulnerability records and the scan report.
// identified by composite key (ID, Report)
// Since each scan report has a separate UUID, the composite key
// would ensure that the immutability of the historical scan reports is guaranteed.
// It is sufficient to store the int64 VulnerabilityRecord Id since the vulnerability records
// are uniquely identified in the table based on the ScannerID and the CVEID
type ReportVulnerabilityRecord struct {
ID int64 `orm:"pk;auto;column(id)"`
Report string `orm:"column(report_uuid);"`
VulnRecordID int64 `orm:"column(vuln_record_id);"`
}
// TableName for VulnerabilityRecord
func (vr *VulnerabilityRecord) TableName() string {
return "vulnerability_record"
}
// TableUnique for VulnerabilityRecord
func (vr *VulnerabilityRecord) TableUnique() [][]string {
return [][]string{
{"cve_id", "registration_uuid", "package", "package_version"},
}
}
// TableName for ReportVulnerabilityRecord
func (rvr *ReportVulnerabilityRecord) TableName() string {
return "report_vulnerability_record"
}
// TableUnique for ReportVulnerabilityRecord
func (rvr *ReportVulnerabilityRecord) TableUnique() [][]string {
return [][]string{
{"report_uuid", "vuln_record_id"},
}
}

6
src/pkg/scan/dao/scan/report_test.go
View File

@ -15,9 +15,9 @@
package scan
import (
htesting "github.com/goharbor/harbor/src/testing"
"testing"
common "github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
@ -27,7 +27,7 @@ import (
// ReportTestSuite is test suite of testing report DAO.
type ReportTestSuite struct {
suite.Suite
htesting.Suite
dao DAO
}
@ -39,7 +39,7 @@ func TestReport(t *testing.T) {
// SetupSuite prepares env for test suite.
func (suite *ReportTestSuite) SetupSuite() {
common.PrepareTestForPostgresSQL()
suite.Suite.SetupSuite()
suite.dao = New()
}

240
src/pkg/scan/dao/scan/vulnerability.go Normal file
View File

@ -0,0 +1,240 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
)
func init() {
orm.RegisterModel(new(VulnerabilityRecord), new(ReportVulnerabilityRecord))
}
// VulnerabilityRecordDao exposes the DAO layer contract to perform
// CRUD operations on vulnerability record objects
type VulnerabilityRecordDao interface {
// Create creates a new vulnerability record
Create(ctx context.Context, vr *VulnerabilityRecord) (int64, error)
// Delete deletes a vulnerability record
Delete(ctx context.Context, vr *VulnerabilityRecord) error
// List lists the vulnerability records
List(ctx context.Context, query *q.Query) ([]*VulnerabilityRecord, error)
// InsertForReport inserts vulnerability records for a report
InsertForReport(ctx context.Context, reportUUID string, vr *VulnerabilityRecord) (int64, error)
// GetForReport gets vulnerability records for a report
GetForReport(ctx context.Context, reportUUID string) ([]*VulnerabilityRecord, error)
// GetForScanner gets vulnerability records for a scanner
GetForScanner(ctx context.Context, registrationUUID string) ([]*VulnerabilityRecord, error)
// DeleteForScanner deletes vulnerability records for a scanner
DeleteForScanner(ctx context.Context, registrationUUID string) (int64, error)
// DeleteForReport deletes vulnerability records for a report
DeleteForReport(ctx context.Context, reportUUID string) (int64, error)
// DeleteForDigests deletes vulnerability records for a provided list of digests
DeleteForDigests(ctx context.Context, digests ...string) (int64, error)
// GetRecordIdsForScanner gets record ids of vulnerability records for a scanner
GetRecordIdsForScanner(ctx context.Context, registrationUUID string) ([]int, error)
}
// NewVulnerabilityRecordDao returns a new dao to handle vulnerability data
func NewVulnerabilityRecordDao() VulnerabilityRecordDao {
return &vulnerabilityRecordDao{}
}
type vulnerabilityRecordDao struct{}
// Create creates new vulnerability record.
func (v *vulnerabilityRecordDao) Create(ctx context.Context, vr *VulnerabilityRecord) (int64, error) {
o, err := orm.FromContext(ctx)
var vrID int64
err = orm.WithTransaction(func(ctx context.Context) error {
var err error
vrID, err = o.InsertOrUpdate(vr, "cve_id, registration_uuid, package, package_version")
return orm.WrapConflictError(err, "vulnerability already exists")
})(ctx)
if errors.IsConflictErr(err) {
if err := o.Read(vr, "cve_id", "registration_uuid", "package", "package_version"); err != nil {
return 0, err
}
return vr.ID, nil
}
return vrID, err
}
// Delete deletes a vulnerability record
func (v *vulnerabilityRecordDao) Delete(ctx context.Context, vr *VulnerabilityRecord) error {
o, err := orm.FromContext(ctx)
if err != nil {
return err
}
_, err = o.Delete(vr, "CVEID", "RegistrationUUID", "Package", "PackageVersion")
return err
}
// List lists the vulnerability records with given query parameters.
// Keywords in query here will be enforced with `exact` way.
// If the registration ID (which = the scanner ID is not specified), the results
// would contain duplicate records for a CVE depending upon the number of registered
// scanners which individually store data about the CVE. In such cases, it is the
// responsibility of the calling code to de-duplicate the CVE records or bucket them
// per registered scanner
func (v *vulnerabilityRecordDao) List(ctx context.Context, query *q.Query) ([]*VulnerabilityRecord, error) {
o, err := orm.FromContext(ctx)
if err != nil {
return nil, err
}
qt := o.QueryTable(new(VulnerabilityRecord))
if query != nil {
if len(query.Keywords) > 0 {
for k, v := range query.Keywords {
if vv, ok := v.([]interface{}); ok {
qt = qt.Filter(fmt.Sprintf("%s__in", k), vv...)
continue
}
qt = qt.Filter(k, v)
}
}
if query.PageNumber > 0 && query.PageSize > 0 {
qt = qt.Limit(query.PageSize, (query.PageNumber-1)*query.PageSize)
}
}
l := make([]*VulnerabilityRecord, 0)
_, err = qt.All(&l)
return l, err
}
// InsertForReport inserts a vulnerability record in the context of scan report
func (v *vulnerabilityRecordDao) InsertForReport(ctx context.Context, reportUUID string, vr *VulnerabilityRecord) (int64, error) {
vrID, err := v.Create(ctx, vr)
if err != nil {
return 0, err
}
rvr := new(ReportVulnerabilityRecord)
rvr.Report = reportUUID
rvr.VulnRecordID = vrID
o, err := orm.FromContext(ctx)
if err != nil {
return 0, err
}
_, rvrID, err := o.ReadOrCreate(rvr, "report_uuid", "vuln_record_id")
return rvrID, err
}
// DeleteForReport deletes the vulnerability records for a single report
func (v *vulnerabilityRecordDao) DeleteForReport(ctx context.Context, reportUUID string) (int64, error) {
o, err := orm.FromContext(ctx)
if err != nil {
return 0, err
}
delCount, err := o.Delete(&ReportVulnerabilityRecord{Report: reportUUID}, "report_uuid")
return delCount, err
}
// GetForReport gets all the vulnerability records for a report based on UUID
func (v *vulnerabilityRecordDao) GetForReport(ctx context.Context, reportUUID string) ([]*VulnerabilityRecord, error) {
vulnRecs := make([]*VulnerabilityRecord, 0)
o, err := orm.FromContext(ctx)
if err != nil {
return nil, err
}
query := `select vulnerability_record.* from vulnerability_record
inner join report_vulnerability_record on
vulnerability_record.id = report_vulnerability_record.vuln_record_id and report_vulnerability_record.report_uuid=?`
_, err = o.Raw(query, reportUUID).QueryRows(&vulnRecs)
return vulnRecs, err
}
// GetForScanner gets all the vulnerability records known to a scanner
// identified by registrationUUID
func (v *vulnerabilityRecordDao) GetForScanner(ctx context.Context, registrationUUID string) ([]*VulnerabilityRecord, error) {
var vulnRecs []*VulnerabilityRecord
o, err := orm.FromContext(ctx)
if err != nil {
return nil, err
}
vulRec := new(VulnerabilityRecord)
qs := o.QueryTable(vulRec)
_, err = qs.Filter("registration_uuid", registrationUUID).All(&vulnRecs)
if err != nil {
return nil, err
}
return vulnRecs, nil
}
// DeleteForScanner deletes all the vulnerability records for a given scanner
// identified by registrationUUID
func (v *vulnerabilityRecordDao) DeleteForScanner(ctx context.Context, registrationUUID string) (int64, error) {
o, err := orm.FromContext(ctx)
if err != nil {
return 0, err
}
vulnRec := new(VulnerabilityRecord)
vulnRec.RegistrationUUID = registrationUUID
return o.Delete(vulnRec, "registration_uuid")
}
// DeleteForDigests deletes the report vulnerability record mappings for the provided
// set of digests
func (v *vulnerabilityRecordDao) DeleteForDigests(ctx context.Context, digests ...string) (int64, error) {
reportDao := New()
ol := q.OrList{}
for _, digest := range digests {
ol.Values = append(ol.Values, digest)
}
reports, err := reportDao.List(ctx, &q.Query{Keywords: q.KeyWords{"digest": &ol}})
if err != nil {
return 0, err
}
numRowsDeleted := int64(0)
for _, report := range reports {
delCount, err := v.DeleteForReport(ctx, report.UUID)
if err != nil {
return 0, err
}
numRowsDeleted += delCount
}
return numRowsDeleted, nil
}
// GetRecordIdsForScanner retrieves the internal Ids of the vulnerability records for a given scanner
// identified by registrationUUID
func (v *vulnerabilityRecordDao) GetRecordIdsForScanner(ctx context.Context, registrationUUID string) ([]int, error) {
vulnRecordIds := make([]int, 0)
o, err := orm.FromContext(ctx)
if err != nil {
return nil, err
}
_, err = o.Raw("select id from vulnerability_record where registration_uuid = ?", registrationUUID).QueryRows(&vulnRecordIds)
if err != nil {
return vulnRecordIds, err
}
return vulnRecordIds, err
}

296
src/pkg/scan/dao/scan/vulnerability_test.go Normal file
View File

@ -0,0 +1,296 @@
package scan
import (
"fmt"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/rest/v1"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"testing"
)
const sampleReportWithCompleteVulnData = `{
"generated_at": "2020-08-01T18:28:49.072885592Z",
"artifact": {
"repository": "library/ubuntu",
"digest": "sha256:d5b40885539615b9aeb7119516427959a158386af13e00d79a7da43ad1b3fb87",
"mime_type": "application/vnd.docker.distribution.manifest.v2+json"
},
"scanner": {
"name": "Trivy",
"vendor": "Aqua Security",
"version": "v0.9.1"
},
"severity": "Medium",
"vulnerabilities": [
{
"id": "CVE-2019-18276",
"package": "bash",
"version": "5.0-6ubuntu1.1",
"severity": "Low",
"description": "An issue was discovered in disable_priv_mode in shell.c in GNU Bash through 5.0 patch 11. By default, if Bash is run with its effective UID not equal to its real UID, it will drop privileges by setting its effective UID to its real UID. However, it does so incorrectly. On Linux and other systems that support \"saved UID\" functionality, the saved UID is not dropped. An attacker with command execution in the shell can use \"enable -f\" for runtime loading of a new builtin, which can be a shared object that calls setuid() and therefore regains privileges. However, binaries running with an effective UID of 0 are unaffected.",
"links": [
"http://packetstormsecurity.com/files/155498/Bash-5.0-Patch-11-Privilege-Escalation.html",
"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-18276",
"https://github.com/bminor/bash/commit/951bdaad7a18cc0dc1036bba86b18b90874d39ff",
"https://security.netapp.com/advisory/ntap-20200430-0003/",
"https://www.youtube.com/watch?v=-wGtxJ8opa8"
],
"layer": {
"digest": "sha256:4739cd2f4f486596c583c79f6033f1a9dee019389d512603609494678c8ccd53",
"diff_id": "sha256:f66829086c450acd5f67d0529a58f7120926c890f04e17aa7f0e9365da86480a"
},
"cwe_ids": ["CWE-476", "CWE-345"],
"cvss":{
"score_v3": 3.2,
"score_v2": 2.3,
"vector_v3": "CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N",
"vector_v2": "AV:L/AC:M/Au:N/C:P/I:N/A:N"
},
"vendor_attributes":[ {
"key": "foo",
"value": "bar"
},
{
"key": "foo1",
"value": "bar1"
}
]
}
]
}`
// VulnerabilityTestSuite is test suite of testing vulnerability DAO.
type VulnerabilityTestSuite struct {
htesting.Suite
rpUUID string
vulnerabilityRecordDao VulnerabilityRecordDao
dao DAO
registrationID string
}
// TestVulnerabilityItem is the entry of ReportTestSuite.
func TestVulnerabilityItem(t *testing.T) {
suite.Run(t, &VulnerabilityTestSuite{})
}
// SetupSuite prepares env for test suite.
func (suite *VulnerabilityTestSuite) SetupSuite() {
suite.Suite.SetupSuite()
suite.rpUUID = "uuid"
suite.vulnerabilityRecordDao = NewVulnerabilityRecordDao()
suite.dao = New()
}
// SetupTest prepares env for test case.
func (suite *VulnerabilityTestSuite) SetupTest() {
r := &Report{
UUID: "uuid",
Digest: "digest1001",
RegistrationUUID: "scannerId1",
MimeType: v1.MimeTypeNativeReport,
Status: job.PendingStatus.String(),
Report: sampleReportWithCompleteVulnData,
}
suite.registerScanner(r.RegistrationUUID)
suite.createReport(r)
vulns := generateVulnerabilityRecordsForReport("uuid", "scannerId1", 10)
for _, v := range vulns {
suite.insertVulnRecordForReport("uuid", v)
}
}
// TearDownTest clears enf for test case.
func (suite *VulnerabilityTestSuite) TearDownTest() {
registrations, err := scanner.ListRegistrations(&q.Query{})
require.NoError(suite.T(), err, "Failed to cleanup scanner registrations")
for _, registration := range registrations {
err = scanner.DeleteRegistration(registration.UUID)
require.NoError(suite.T(), err, "Error when cleaning up scanner registrations")
}
reports, err := suite.dao.List(orm.Context(), &q.Query{})
require.NoError(suite.T(), err)
for _, report := range reports {
suite.cleanUpAdditionalData(report.UUID, report.RegistrationUUID)
}
}
// TestVulnerabilityRecordsListForReport tests listing of vulnerability record for reports
func (suite *VulnerabilityTestSuite) TestVulnerabilityRecordsListForReport() {
// create a second report and associate the same vulnerability record set to the report
r := &Report{
UUID: "uuid1",
Digest: "digest1002",
RegistrationUUID: "scannerId2",
MimeType: v1.MimeTypeNativeReport,
Status: job.PendingStatus.String(),
Report: sampleReportWithCompleteVulnData,
}
suite.registerScanner(r.RegistrationUUID)
suite.createReport(r)
// insert a set of vulnerability records for this report. the vulnerability records
// belong to the same scanner
vulns := generateVulnerabilityRecordsForReport("uuid1", "scannerId2", 10)
for _, v := range vulns {
suite.insertVulnRecordForReport("uuid1", v)
}
// fetch the records for the first report. Additionally assert that these records
// indeed belong to the same report being fetched and not to another report
{
vulns, err := suite.vulnerabilityRecordDao.GetForReport(orm.Context(), "uuid")
require.NoError(suite.T(), err, "Error when fetching vulnerability records for report")
require.True(suite.T(), len(vulns) > 0)
}
{
vulns, err := suite.vulnerabilityRecordDao.GetForReport(orm.Context(), "uuid1")
require.NoError(suite.T(), err, "Error when fetching vulnerability records for report")
require.True(suite.T(), len(vulns) > 0)
}
}
// TestGetVulnerabilityRecordsForScanner gets vulnerability records for scanner
func (suite *VulnerabilityTestSuite) TestGetVulnerabilityRecordsForScanner() {
vulns, err := suite.vulnerabilityRecordDao.GetForScanner(orm.Context(), "scannerId1")
require.NoError(suite.T(), err, "Error when fetching vulnerability records for report")
require.True(suite.T(), len(vulns) > 0)
}
// TestGetVulnerabilityRecordIdsForScanner gets vulnerability records for scanner
func (suite *VulnerabilityTestSuite) TestGetVulnerabilityRecordIdsForScanner() {
vulns, err := suite.vulnerabilityRecordDao.GetRecordIdsForScanner(orm.Context(), "scannerId1")
require.NoError(suite.T(), err, "Error when fetching vulnerability records for report")
require.True(suite.T(), len(vulns) > 0)
}
// TestDeleteForDigest tests deleting vulnerability report for a specific digest
func (suite *VulnerabilityTestSuite) TestDeleteForDigest() {
// create a second report and associate the same vulnerability record set to the report
r := &Report{
UUID: "uuid1",
Digest: "digest1",
RegistrationUUID: "scannerId2",
MimeType: v1.MimeTypeNativeReport,
Status: job.PendingStatus.String(),
Report: sampleReportWithCompleteVulnData,
}
suite.registerScanner(r.RegistrationUUID)
suite.createReport(r)
// insert a set of vulnerability records for this report. the vulnerability records
// belong to the same scanner
vulns := generateVulnerabilityRecordsForReport("uuid1", "scannerId2", 10)
for _, v := range vulns {
suite.insertVulnRecordForReport("uuid1", v)
}
delCount, err := suite.vulnerabilityRecordDao.DeleteForDigests(orm.Context(), "digest1")
require.NoError(suite.T(), err)
assert.Equal(suite.T(), int64(10), delCount)
}
func (suite *VulnerabilityTestSuite) TestDuplicateRecords() {
r := &Report{
UUID: "uuid2",
Digest: "digest1002",
RegistrationUUID: "scannerId1",
MimeType: v1.MimeTypeNativeReport,
Status: job.PendingStatus.String(),
Report: sampleReportWithCompleteVulnData,
}
suite.createReport(r)
vulns := generateVulnerabilityRecordsForReport("uuid2", "scannerId1", 10)
for _, v := range vulns {
suite.insertVulnRecordForReport("uuid2", v)
}
}
// TestDeleteVulnerabilityRecord gets vulnerability records for scanner
func (suite *VulnerabilityTestSuite) TestDeleteVulnerabilityRecord() {
vulns, err := suite.vulnerabilityRecordDao.GetForScanner(orm.Context(), "scannerId1")
require.NoError(suite.T(), err, "Error when fetching vulnerability records for report")
require.True(suite.T(), len(vulns) > 0)
for _, vuln := range vulns {
err = suite.vulnerabilityRecordDao.Delete(orm.Context(), vuln)
require.NoError(suite.T(), err)
}
}
// TestListVulnerabilityRecord gets vulnerability records for scanner
func (suite *VulnerabilityTestSuite) TestListVulnerabilityRecord() {
vulns, err := suite.vulnerabilityRecordDao.List(orm.Context(), &q.Query{Keywords: map[string]interface{}{"CVEID": "CVE-ID1"}})
require.NoError(suite.T(), err, "Error when fetching vulnerability records for report")
require.True(suite.T(), len(vulns) > 0)
}
func (suite *VulnerabilityTestSuite) createReport(r *Report) {
id, err := suite.dao.Create(orm.Context(), r)
require.NoError(suite.T(), err)
require.Condition(suite.T(), func() (success bool) {
success = id > 0
return
})
}
func (suite *VulnerabilityTestSuite) insertVulnRecordForReport(reportUUID string, vr *VulnerabilityRecord) {
id, err := suite.vulnerabilityRecordDao.InsertForReport(orm.Context(), reportUUID, vr)
require.NoError(suite.T(), err)
require.True(suite.T(), id > 0, "Failed to insert vulnerability record row for report %s", reportUUID)
}
func (suite *VulnerabilityTestSuite) cleanUpAdditionalData(reportID string, scannerID string) {
_, err := suite.dao.DeleteMany(orm.Context(), q.Query{Keywords: q.KeyWords{"uuid": reportID}})
require.NoError(suite.T(), err)
_, err = suite.vulnerabilityRecordDao.DeleteForReport(orm.Context(), reportID)
require.NoError(suite.T(), err, "Failed to cleanup records")
_, err = suite.vulnerabilityRecordDao.DeleteForScanner(orm.Context(), scannerID)
require.NoError(suite.T(), err, "Failed to delete vulnerability records")
}
func (suite *VulnerabilityTestSuite) registerScanner(registrationUUID string) {
r := &scanner.Registration{
UUID: registrationUUID,
Name: registrationUUID,
Description: "sample registration",
URL: fmt.Sprintf("https://sample.scanner.com/%s", registrationUUID),
}
_, err := scanner.AddRegistration(r)
require.NoError(suite.T(), err, "add new registration")
}
func generateVulnerabilityRecordsForReport(reportUUID string, registrationUUID string, numRecords int) []*VulnerabilityRecord {
vulns := make([]*VulnerabilityRecord, 0)
for i := 1; i <= numRecords; i++ {
vulnV2 := new(VulnerabilityRecord)
vulnV2.CVEID = fmt.Sprintf("CVE-ID%d", i)
vulnV2.Package = fmt.Sprintf("Package%d", i)
vulnV2.PackageVersion = "NotAvailable"
vulnV2.PackageType = "Unknown"
vulnV2.Fix = "1.0.0"
vulnV2.URLs = "url1"
vulnV2.RegistrationUUID = registrationUUID
if i%2 == 0 {
vulnV2.Severity = "High"
} else if i%3 == 0 {
vulnV2.Severity = "Medium"
} else if i%4 == 0 {
vulnV2.Severity = "Critical"
} else {
vulnV2.Severity = "Low"
}
vulns = append(vulns, vulnV2)
}
return vulns
}

4
src/pkg/scan/job_test.go
View File

@ -103,7 +103,7 @@ func (suite *JobTestSuite) TestJob() {
robotData, err := robot.ToJSON()
require.NoError(suite.T(), err)
mimeTypes := []string{v1.MimeTypeNativeReport}
mimeTypes := []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport}
jp := make(job.Parameters)
jp[JobParamRegistration] = rData
@ -142,7 +142,7 @@ func (suite *JobTestSuite) TestJob() {
jRep, err := json.Marshal(rp)
require.NoError(suite.T(), err)
mc.On("GetScanReport", "scan_id", v1.MimeTypeNativeReport).Return(string(jRep), nil)
mc.On("GetScanReport", "scan_id", v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport).Return(string(jRep), nil)
mocktesting.OnAnything(suite.mcp, "Get").Return(mc, nil)
crp := &CheckInReport{

198
src/pkg/scan/postprocessors/report_converters.go Normal file
View File

@ -0,0 +1,198 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postprocessors
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
)
// NativeScanReportConverter is an interface that establishes the contract for the conversion process of a harbor native vulnerability report
// It is the responsibility of the implementation to store the report in a manner easily retrievable using the
// report UUID
type NativeScanReportConverter interface {
ToRelationalSchema(ctx context.Context, reportUUID string, registrationUUID string, digest string, reportData string) (string, string, error)
FromRelationalSchema(ctx context.Context, reportUUID string, artifactDigest string, reportSummary string) (string, error)
}
// nativeToRelationalSchemaConverter is responsible for converting the JSON scan report from the Harbor 1.0 format to
// the generic vulnerability format which follows a normalized storage schema.
type nativeToRelationalSchemaConverter struct {
dao scan.VulnerabilityRecordDao
}
// NewNativeToRelationalSchemaConverter returns a new instance of a V1 report to V2 report converter
func NewNativeToRelationalSchemaConverter() NativeScanReportConverter {
return &nativeToRelationalSchemaConverter{dao: scan.NewVulnerabilityRecordDao()}
}
// ToRelationalSchema converts the vulnerability report data present as JSON to the new relational VulnerabilityRecord instance
func (c *nativeToRelationalSchemaConverter) ToRelationalSchema(ctx context.Context, reportUUID string, registrationUUID string, digest string, reportData string) (string, string, error) {
if len(reportData) == 0 {
log.Infof("There is no vulnerability report to toSchema for report UUID : %s", reportUUID)
return reportUUID, "", nil
}
// parse the raw report with the V1 schema of the report to the normalized structures
rawReport := new(vuln.Report)
if err := json.Unmarshal([]byte(reportData), &rawReport); err != nil {
return "", "", errors.Wrap(err, fmt.Sprintf("Error when toSchema V1 report to V2"))
}
if err := c.toSchema(ctx, reportUUID, registrationUUID, digest, reportData); err != nil {
return "", "", errors.Wrap(err, fmt.Sprintf("Error when converting vulnerability report"))
}
rawReport.Vulnerabilities = nil
data, err := json.Marshal(rawReport)
if err != nil {
return "", "", errors.Wrap(err, fmt.Sprintf("Error when persisting raw report summary"))
}
return reportUUID, string(data), nil
}
// FromRelationalSchema converts the generic vulnerability record stored in relational form to the
// native JSON blob.
func (c *nativeToRelationalSchemaConverter) FromRelationalSchema(ctx context.Context, reportUUID string, artifactDigest string, reportSummary string) (string, error) {
vulns, err := c.dao.GetForReport(ctx, reportUUID)
if err != nil {
return "", errors.Wrap(err, fmt.Sprintf("Error when toSchema generic vulnerability records for %s", reportUUID))
}
rp, err := c.fromSchema(ctx, reportUUID, artifactDigest, reportSummary, vulns)
if err != nil {
return "", err
}
return rp, nil
}
func (c *nativeToRelationalSchemaConverter) toSchema(ctx context.Context, reportUUID string, registrationUUID string, digest string, rawReportData string) error {
var vulnReport vuln.Report
err := json.Unmarshal([]byte(rawReportData), &vulnReport)
if err != nil {
return err
}
for _, v := range vulnReport.Vulnerabilities {
vulnV2 := new(scan.VulnerabilityRecord)
vulnV2.CVEID = v.ID
vulnV2.Description = v.Description
vulnV2.Package = v.Package
vulnV2.PackageVersion = v.Version
vulnV2.PackageType = "Unknown"
vulnV2.Fix = v.FixVersion
vulnV2.URLs = strings.Join(v.Links, "|")
vulnV2.RegistrationUUID = registrationUUID
vulnV2.Severity = v.Severity.String()
// process the CVSS scores if the data is available
if (vuln.CVSS{} != v.CVSSDetails) {
vulnV2.CVE3Score = v.CVSSDetails.ScoreV3
vulnV2.CVE2Score = v.CVSSDetails.ScoreV2
vulnV2.CVSS3Vector = v.CVSSDetails.VectorV3
vulnV2.CVSS2Vector = v.CVSSDetails.VectorV2
}
if len(v.CWEIds) > 0 {
vulnV2.CWEIDs = strings.Join(v.CWEIds, ",")
}
// marshall the presented vendor attributes as a json string
if len(v.VendorAttributes) > 0 {
vendorAttributes, err := json.Marshal(v.VendorAttributes)
// set the vendor attributes iff unmarshalling is successful
if err == nil {
vulnV2.VendorAttributes = string(vendorAttributes)
}
}
_, err = c.dao.InsertForReport(ctx, reportUUID, vulnV2)
if err != nil {
log.Warningf("Could not insert vulnerability record - report: %s, cve_id: %s, scanner: %s, package: %s, package_version: %s", reportUUID, v.ID, registrationUUID, v.Package, v.Version)
}
}
log.Infof("Converted %d vulnerability records to the new schema for report ID %s and scanner Id %s", len(vulnReport.Vulnerabilities), reportUUID, registrationUUID)
return nil
}
func (c *nativeToRelationalSchemaConverter) fromSchema(ctx context.Context, reportUUID string, artifactDigest string, reportSummary string, records []*scan.VulnerabilityRecord) (string, error) {
if len(reportSummary) == 0 {
return "", nil
}
vulnerabilityItems := make([]*vuln.VulnerabilityItem, 0)
for _, record := range records {
vi := new(vuln.VulnerabilityItem)
vi.ID = record.CVEID
vi.ArtifactDigest = artifactDigest
vi.CVSSDetails.ScoreV2 = record.CVE2Score
vi.CVSSDetails.ScoreV3 = record.CVE3Score
vi.CVSSDetails.VectorV2 = record.CVSS2Vector
vi.CVSSDetails.VectorV3 = record.CVSS3Vector
cweIDs := strings.Split(record.CWEIDs, ",")
for _, cweID := range cweIDs {
vi.CWEIds = append(vi.CWEIds, cweID)
}
vi.CWEIds = cweIDs
vi.Description = record.Description
vi.FixVersion = record.Fix
vi.Version = record.PackageVersion
urls := strings.Split(record.URLs, "|")
for _, url := range urls {
vi.Links = append(vi.Links, url)
}
vi.Severity = vuln.ParseSeverityVersion3(record.Severity)
vi.Package = record.Package
var vendorAttributes map[string]interface{}
_ = json.Unmarshal([]byte(record.VendorAttributes), &vendorAttributes)
vi.VendorAttributes = vendorAttributes
vulnerabilityItems = append(vulnerabilityItems, vi)
}
rp := new(vuln.Report)
err := json.Unmarshal([]byte(reportSummary), rp)
if err != nil {
return "", err
}
if len(vulnerabilityItems) > 0 {
rp.Vulnerabilities = make([]*vuln.VulnerabilityItem, 0)
for _, v := range vulnerabilityItems {
rp.Vulnerabilities = append(rp.Vulnerabilities, v)
}
}
data, err := json.Marshal(rp)
if err != nil {
return "", err
}
return string(data), nil
}
// GetNativeV1ReportFromResolvedData returns the native V1 scan report from the resolved
// interface data.
func (c *nativeToRelationalSchemaConverter) getNativeV1ReportFromResolvedData(ctx job.Context, rp interface{}) (*vuln.Report, error) {
report, ok := rp.(*vuln.Report)
if !ok {
return nil, errors.New("Data cannot be converted to v1 report format")
}
ctx.GetLogger().Infof("Converted raw data to report. Count of Vulnerabilities in report : %d", len(report.Vulnerabilities))
return report, nil
}

496
src/pkg/scan/postprocessors/report_converters_test.go Normal file
View File

@ -0,0 +1,496 @@
package postprocessors
import (
"encoding/json"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/report"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"testing"
"time"
)
const sampleReport = `{
"generated_at": "2020-08-01T18:28:49.072885592Z",
"artifact": {
"repository": "library/ubuntu",
"digest": "sha256:d5b40885539615b9aeb7119516427959a158386af13e00d79a7da43ad1b3fb87",
"mime_type": "application/vnd.docker.distribution.manifest.v2+json"
},
"scanner": {
"name": "Trivy",
"vendor": "Aqua Security",
"version": "v0.9.1"
},
"severity": "Medium",
"vulnerabilities": [
{
"id": "CVE-2019-18276",
"package": "bash",
"version": "5.0-6ubuntu1.1",
"severity": "Low",
"description": "An issue was discovered in disable_priv_mode in shell.c in GNU Bash through 5.0 patch 11. By default, if Bash is run with its effective UID not equal to its real UID, it will drop privileges by setting its effective UID to its real UID. However, it does so incorrectly. On Linux and other systems that support \"saved UID\" functionality, the saved UID is not dropped. An attacker with command execution in the shell can use \"enable -f\" for runtime loading of a new builtin, which can be a shared object that calls setuid() and therefore regains privileges. However, binaries running with an effective UID of 0 are unaffected.",
"links": [
"http://packetstormsecurity.com/files/155498/Bash-5.0-Patch-11-Privilege-Escalation.html",
"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-18276",
"https://github.com/bminor/bash/commit/951bdaad7a18cc0dc1036bba86b18b90874d39ff",
"https://security.netapp.com/advisory/ntap-20200430-0003/",
"https://www.youtube.com/watch?v=-wGtxJ8opa8"
],
"layer": {
"digest": "sha256:4739cd2f4f486596c583c79f6033f1a9dee019389d512603609494678c8ccd53",
"diff_id": "sha256:f66829086c450acd5f67d0529a58f7120926c890f04e17aa7f0e9365da86480a"
}
}
]
}`
const sampleReportWithCWEAndCVSS = `{
"generated_at": "2020-08-01T18:28:49.072885592Z",
"artifact": {
"repository": "library/ubuntu",
"digest": "sha256:d5b40885539615b9aeb7119516427959a158386af13e00d79a7da43ad1b3fb87",
"mime_type": "application/vnd.docker.distribution.manifest.v2+json"
},
"scanner": {
"name": "Trivy",
"vendor": "Aqua Security",
"version": "v0.9.1"
},
"severity": "Medium",
"vulnerabilities": [
{
"id": "CVE-2019-18276",
"package": "bash",
"version": "5.0-6ubuntu1.1",
"severity": "Low",
"description": "An issue was discovered in disable_priv_mode in shell.c in GNU Bash through 5.0 patch 11. By default, if Bash is run with its effective UID not equal to its real UID, it will drop privileges by setting its effective UID to its real UID. However, it does so incorrectly. On Linux and other systems that support \"saved UID\" functionality, the saved UID is not dropped. An attacker with command execution in the shell can use \"enable -f\" for runtime loading of a new builtin, which can be a shared object that calls setuid() and therefore regains privileges. However, binaries running with an effective UID of 0 are unaffected.",
"links": [
"http://packetstormsecurity.com/files/155498/Bash-5.0-Patch-11-Privilege-Escalation.html",
"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-18276",
"https://github.com/bminor/bash/commit/951bdaad7a18cc0dc1036bba86b18b90874d39ff",
"https://security.netapp.com/advisory/ntap-20200430-0003/",
"https://www.youtube.com/watch?v=-wGtxJ8opa8"
],
"layer": {
"digest": "sha256:4739cd2f4f486596c583c79f6033f1a9dee019389d512603609494678c8ccd53",
"diff_id": "sha256:f66829086c450acd5f67d0529a58f7120926c890f04e17aa7f0e9365da86480a"
},
"cwe_ids": ["CWE-476", "CWE-345"],
"preferred_cvss":{
"score_v3": 3.2,
"score_v2": 2.3,
"vector_v3": "CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N",
"vector_v2": "AV:L/AC:M/Au:N/C:P/I:N/A:N"
}
}
]
}`
const sampleReportWithCompleteVulnData = `{
"generated_at": "2020-08-01T18:28:49.072885592Z",
"artifact": {
"repository": "library/ubuntu",
"digest": "sha256:d5b40885539615b9aeb7119516427959a158386af13e00d79a7da43ad1b3fb87",
"mime_type": "application/vnd.docker.distribution.manifest.v2+json"
},
"scanner": {
"name": "Trivy",
"vendor": "Aqua Security",
"version": "v0.9.1"
},
"severity": "Medium",
"vulnerabilities": [
{
"id": "CVE-2019-18276",
"package": "bash",
"version": "5.0-6ubuntu1.1",
"severity": "Low",
"description": "An issue was discovered in disable_priv_mode in shell.c in GNU Bash through 5.0 patch 11. By default, if Bash is run with its effective UID not equal to its real UID, it will drop privileges by setting its effective UID to its real UID. However, it does so incorrectly. On Linux and other systems that support \"saved UID\" functionality, the saved UID is not dropped. An attacker with command execution in the shell can use \"enable -f\" for runtime loading of a new builtin, which can be a shared object that calls setuid() and therefore regains privileges. However, binaries running with an effective UID of 0 are unaffected.",
"links": [
"http://packetstormsecurity.com/files/155498/Bash-5.0-Patch-11-Privilege-Escalation.html",
"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-18276",
"https://github.com/bminor/bash/commit/951bdaad7a18cc0dc1036bba86b18b90874d39ff",
"https://security.netapp.com/advisory/ntap-20200430-0003/",
"https://www.youtube.com/watch?v=-wGtxJ8opa8"
],
"layer": {
"digest": "sha256:4739cd2f4f486596c583c79f6033f1a9dee019389d512603609494678c8ccd53",
"diff_id": "sha256:f66829086c450acd5f67d0529a58f7120926c890f04e17aa7f0e9365da86480a"
},
"cwe_ids": ["CWE-476", "CWE-345"],
"preferred_cvss":{
"score_v3": 3.2,
"score_v2": 2.3,
"vector_v3": "CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N",
"vector_v2": "AV:L/AC:M/Au:N/C:P/I:N/A:N"
},
"vendor_attributes":{
"CVSS":{
"nvd" : {
"V2Score": 7.1,
"V2Vector": "AV:L/AC:M/Au:N/C:P/I:N/A:N",
"V3Score": 6.5,
"V3Vector":"CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N"
}
}
}
}
]
}`
const sampleReportWithMixedSeverity = `{
"generated_at": "2020-08-01T18:28:49.072885592Z",
"artifact": {
"repository": "library/ubuntu",
"digest": "sha256:d5b40885539615b9aeb7119516427959a158386af13e00d79a7da43ad1b3fb87",
"mime_type": "application/vnd.docker.distribution.manifest.v2+json"
},
"scanner": {
"name": "Trivy",
"vendor": "Aqua Security",
"version": "v0.9.1"
},
"severity": "Medium",
"vulnerabilities": [
{
"id": "CVE-2019-18276",
"package": "bash",
"version": "5.0-6ubuntu1.1",
"severity": "Low",
"description": "An issue was discovered in disable_priv_mode in shell.c in GNU Bash through 5.0 patch 11. By default, if Bash is run with its effective UID not equal to its real UID, it will drop privileges by setting its effective UID to its real UID. However, it does so incorrectly. On Linux and other systems that support \"saved UID\" functionality, the saved UID is not dropped. An attacker with command execution in the shell can use \"enable -f\" for runtime loading of a new builtin, which can be a shared object that calls setuid() and therefore regains privileges. However, binaries running with an effective UID of 0 are unaffected.",
"links": [
"http://packetstormsecurity.com/files/155498/Bash-5.0-Patch-11-Privilege-Escalation.html",
"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-18276",
"https://github.com/bminor/bash/commit/951bdaad7a18cc0dc1036bba86b18b90874d39ff",
"https://security.netapp.com/advisory/ntap-20200430-0003/",
"https://www.youtube.com/watch?v=-wGtxJ8opa8"
],
"layer": {
"digest": "sha256:4739cd2f4f486596c583c79f6033f1a9dee019389d512603609494678c8ccd53",
"diff_id": "sha256:f66829086c450acd5f67d0529a58f7120926c890f04e17aa7f0e9365da86480a"
},
"cwe_ids": ["CWE-476", "CWE-345"],
"preferred_cvss":{
"score_v3": 3.2,
"score_v2": 2.3,
"vector_v3": "CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N",
"vector_v2": "AV:L/AC:M/Au:N/C:P/I:N/A:N"
},
"vendor_attributes":{
"CVSS":{
"nvd" : {
"V2Score": 7.1,
"V2Vector": "AV:L/AC:M/Au:N/C:P/I:N/A:N",
"V3Score": 6.5,
"V3Vector":"CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N"
}
}
}
},
{
"id": "CVE-2013-7445",
"package": "linux",
"version": "4.9.189-3+deb9u2",
"severity": "High",
"description": "An issue was discovered in disable_priv_mode in shell.c in GNU Bash through 5.0 patch 11. By default, if Bash is run with its effective UID not equal to its real UID, it will drop privileges by setting its effective UID to its real UID. However, it does so incorrectly. On Linux and other systems that support \"saved UID\" functionality, the saved UID is not dropped. An attacker with command execution in the shell can use \"enable -f\" for runtime loading of a new builtin, which can be a shared object that calls setuid() and therefore regains privileges. However, binaries running with an effective UID of 0 are unaffected.",
"links": [
"http://packetstormsecurity.com/files/155498/Bash-5.0-Patch-11-Privilege-Escalation.html",
"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-18276",
"https://github.com/bminor/bash/commit/951bdaad7a18cc0dc1036bba86b18b90874d39ff",
"https://security.netapp.com/advisory/ntap-20200430-0003/",
"https://www.youtube.com/watch?v=-wGtxJ8opa8"
],
"layer": {
"digest": "sha256:4739cd2f4f486596c583c79f6033f1a9dee019389d512603609494678c8ccd53",
"diff_id": "sha256:f66829086c450acd5f67d0529a58f7120926c890f04e17aa7f0e9365da86480a"
},
"cwe_ids": ["CWE-476", "CWE-345"],
"preferred_cvss":{
"score_v3": 3.2,
"score_v2": 2.3,
"vector_v3": "CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N",
"vector_v2": "AV:L/AC:M/Au:N/C:P/I:N/A:N"
},
"vendor_attributes":{
"CVSS":{
"nvd" : {
"V2Score": 7.1,
"V2Vector": "AV:L/AC:M/Au:N/C:P/I:N/A:N",
"V3Score": 6.5,
"V3Vector":"CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N"
}
}
}
},
{
"id": "CVE-2019-2182",
"package": "bash",
"version": "5.0-6ubuntu1.1",
"severity": "Medium",
"description": "An issue was discovered in disable_priv_mode in shell.c in GNU Bash through 5.0 patch 11. By default, if Bash is run with its effective UID not equal to its real UID, it will drop privileges by setting its effective UID to its real UID. However, it does so incorrectly. On Linux and other systems that support \"saved UID\" functionality, the saved UID is not dropped. An attacker with command execution in the shell can use \"enable -f\" for runtime loading of a new builtin, which can be a shared object that calls setuid() and therefore regains privileges. However, binaries running with an effective UID of 0 are unaffected.",
"links": [
"http://packetstormsecurity.com/files/155498/Bash-5.0-Patch-11-Privilege-Escalation.html",
"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-18276",
"https://github.com/bminor/bash/commit/951bdaad7a18cc0dc1036bba86b18b90874d39ff",
"https://security.netapp.com/advisory/ntap-20200430-0003/",
"https://www.youtube.com/watch?v=-wGtxJ8opa8"
],
"layer": {
"digest": "sha256:4739cd2f4f486596c583c79f6033f1a9dee019389d512603609494678c8ccd53",
"diff_id": "sha256:f66829086c450acd5f67d0529a58f7120926c890f04e17aa7f0e9365da86480a"
},
"cwe_ids": ["CWE-476", "CWE-345"],
"preferred_cvss":{
"score_v3": 3.2,
"score_v2": 2.3,
"vector_v3": "CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N",
"vector_v2": "AV:L/AC:M/Au:N/C:P/I:N/A:N"
},
"vendor_attributes":{
"CVSS":{
"nvd" : {
"V2Score": 7.1,
"V2Vector": "AV:L/AC:M/Au:N/C:P/I:N/A:N",
"V3Score": 6.5,
"V3Vector":"CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N"
}
}
}
}
]
}`
// TestReportConverterSuite
type TestReportConverterSuite struct {
htesting.Suite
rc NativeScanReportConverter
rpUUID string
vulnerabilityRecordDao scan.VulnerabilityRecordDao
reportDao scan.DAO
registrationID string
}
// SetupTest prepares env for test cases.
func (suite *TestReportConverterSuite) SetupTest() {
suite.rpUUID = "reportUUID"
suite.registrationID = "ruuid"
r := &scanner.Registration{
UUID: suite.registrationID,
Name: "forUT",
Description: "sample registration",
URL: "https://sample.scanner.com",
}
_, err := scanner.AddRegistration(r)
require.NoError(suite.T(), err, "add new registration")
}
// TestReportConverterTests specifies the test suite
func TestReportConverterTests(t *testing.T) {
suite.Run(t, &TestReportConverterSuite{})
}
// SetupSuite sets up the report converter suite test cases
func (suite *TestReportConverterSuite) SetupSuite() {
suite.rc = NewNativeToRelationalSchemaConverter()
suite.Suite.SetupSuite()
suite.vulnerabilityRecordDao = scan.NewVulnerabilityRecordDao()
suite.reportDao = scan.New()
}
// TearDownTest clears test env for test cases.
func (suite *TestReportConverterSuite) TearDownTest() {
// No delete method defined in manager as no requirement,
// so, to clear env, call dao method here
scanner.DeleteRegistration(suite.registrationID)
reports, err := suite.reportDao.List(orm.Context(), &q.Query{})
require.True(suite.T(), err == nil, "Failed to delete vulnerability records")
for _, report := range reports {
_, err := suite.reportDao.DeleteMany(orm.Context(), q.Query{Keywords: q.KeyWords{"uuid": report.UUID}})
require.NoError(suite.T(), err)
_, err = suite.vulnerabilityRecordDao.DeleteForReport(orm.Context(), report.UUID)
require.NoError(suite.T(), err, "Failed to delete vulnerability records")
_, err = suite.vulnerabilityRecordDao.DeleteForScanner(orm.Context(), report.RegistrationUUID)
require.NoError(suite.T(), err, "Failed to delete vulnerability records")
}
}
// TestConvertReport tests the report conversion logic
func (suite *TestReportConverterSuite) TestConvertReport() {
rp := &scan.Report{
Digest: "d1000",
RegistrationUUID: "ruuid",
MimeType: v1.MimeTypeNativeReport,
Report: sampleReport,
StartTime: time.Now(),
EndTime: time.Now().Add(1000),
UUID: "reportUUID",
}
suite.create(rp)
ruuid, summary, err := suite.rc.ToRelationalSchema(orm.Context(), rp.UUID, rp.RegistrationUUID, rp.Digest, rp.Report)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), rp.UUID, ruuid)
suite.validateReportSummary(summary, sampleReport)
}
// TestConvertReportWithCWEAndCVSS tests the report conversion with CVSS and CWE information
func (suite *TestReportConverterSuite) TestConvertReportWithCWEAndCVSS() {
rp := &scan.Report{
Digest: "d1000",
RegistrationUUID: "ruuid",
MimeType: v1.MimeTypeNativeReport,
Report: sampleReportWithCWEAndCVSS,
StartTime: time.Now(),
EndTime: time.Now().Add(1000),
UUID: "reportUUID1",
}
suite.create(rp)
ruuid, summary, err := suite.rc.ToRelationalSchema(orm.Context(), rp.UUID, rp.RegistrationUUID, rp.Digest, rp.Report)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), rp.UUID, ruuid)
suite.validateReportSummary(summary, sampleReportWithCWEAndCVSS)
}
// TestConvertReportWithCompleteVulnData tests report conversion with complete vulnerability data
func (suite *TestReportConverterSuite) TestConvertReportWithCompleteVulnData() {
rp := &scan.Report{
Digest: "d1000",
RegistrationUUID: "ruuid",
MimeType: v1.MimeTypeNativeReport,
Report: sampleReportWithCompleteVulnData,
StartTime: time.Now(),
EndTime: time.Now().Add(1000),
UUID: "reportUUID2",
}
suite.create(rp)
ruuid, summary, err := suite.rc.ToRelationalSchema(orm.Context(), rp.UUID, rp.RegistrationUUID, rp.Digest, rp.Report)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), rp.UUID, ruuid)
suite.validateReportSummary(summary, sampleReportWithCompleteVulnData)
}
func (suite *TestReportConverterSuite) TestConvertToNativeReport() {
rp := &scan.Report{
Digest: "d1000",
RegistrationUUID: "ruuid",
MimeType: v1.MimeTypeNativeReport,
Report: sampleReportWithCompleteVulnData,
StartTime: time.Now(),
EndTime: time.Now().Add(1000),
UUID: "reportUUID2",
}
suite.create(rp)
_, summary, err := suite.rc.ToRelationalSchema(orm.Context(), rp.UUID, rp.RegistrationUUID, rp.Digest, sampleReportWithCompleteVulnData)
completeReport, err := suite.rc.FromRelationalSchema(orm.Context(), rp.UUID, rp.Digest, summary)
require.NoError(suite.T(), err)
v := new(vuln.Report)
err = json.Unmarshal([]byte(sampleReportWithCompleteVulnData), v)
require.NoError(suite.T(), err)
v.WithArtifactDigest(rp.Digest)
data, err := json.Marshal(v)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), string(data), completeReport)
}
func (suite *TestReportConverterSuite) TestNativeReportSummaryAfterConversion() {
rp := &scan.Report{
Digest: "d1000",
RegistrationUUID: "ruuid",
MimeType: v1.MimeTypeGenericVulnerabilityReport,
Report: sampleReportWithMixedSeverity,
StartTime: time.Now(),
EndTime: time.Now().Add(1000),
UUID: "reportUUID2",
}
suite.create(rp)
_, summary, err := suite.rc.ToRelationalSchema(orm.Context(), rp.UUID, rp.RegistrationUUID, rp.Digest, rp.Report)
require.NoError(suite.T(), err)
completeReport, err := suite.rc.FromRelationalSchema(orm.Context(), rp.UUID, rp.Digest, summary)
require.NoError(suite.T(), err)
v := new(vuln.Report)
err = json.Unmarshal([]byte(rp.Report), v)
require.NoError(suite.T(), err)
v.WithArtifactDigest(rp.Digest)
data, err := json.Marshal(v)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), string(data), completeReport)
// validate that summarization happens correctly on this report
rp.Report = completeReport
rp.Status = job.SuccessStatus.String()
summ, err := report.GenerateSummary(rp)
require.NoError(suite.T(), err)
nativeReportSummary := summ.(*vuln.NativeReportSummary)
sevMapping := nativeReportSummary.Summary.Summary
assert.True(suite.T(), len(sevMapping) == 3, "Expected entries in severity mapping for 'High', 'Low', 'Medium'")
assert.Equal(suite.T(), 1, sevMapping[vuln.Low])
assert.Equal(suite.T(), 1, sevMapping[vuln.High])
assert.Equal(suite.T(), 1, sevMapping[vuln.Medium])
}
func (suite *TestReportConverterSuite) TestGenericVulnReportSummaryAfterConversion() {
rp := &scan.Report{
Digest: "d1000",
RegistrationUUID: "ruuid",
MimeType: v1.MimeTypeNativeReport,
Report: sampleReportWithMixedSeverity,
StartTime: time.Now(),
EndTime: time.Now().Add(1000),
UUID: "reportUUID2",
}
suite.create(rp)
_, summary, err := suite.rc.ToRelationalSchema(orm.Context(), rp.UUID, rp.RegistrationUUID, rp.Digest, rp.Report)
require.NoError(suite.T(), err)
completeReport, err := suite.rc.FromRelationalSchema(orm.Context(), rp.UUID, rp.Digest, summary)
require.NoError(suite.T(), err)
v := new(vuln.Report)
err = json.Unmarshal([]byte(rp.Report), v)
require.NoError(suite.T(), err)
v.WithArtifactDigest(rp.Digest)
data, err := json.Marshal(v)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), string(data), completeReport)
// validate that summarization happens correctly on this report
rp.Report = completeReport
rp.Status = job.SuccessStatus.String()
summ, err := report.GenerateSummary(rp)
require.NoError(suite.T(), err)
nativeReportSummary := summ.(*vuln.NativeReportSummary)
sevMapping := nativeReportSummary.Summary.Summary
assert.True(suite.T(), len(sevMapping) == 3, "Expected entries in severity mapping for 'High', 'Low', 'Medium'")
assert.Equal(suite.T(), 1, sevMapping[vuln.Low])
assert.Equal(suite.T(), 1, sevMapping[vuln.High])
assert.Equal(suite.T(), 1, sevMapping[vuln.Medium])
}
func (suite *TestReportConverterSuite) create(r *scan.Report) {
id, err := suite.reportDao.Create(orm.Context(), r)
require.NoError(suite.T(), err)
require.Condition(suite.T(), func() (success bool) {
success = id > 0
return
})
}
func (suite *TestReportConverterSuite) validateReportSummary(summary string, rawReport string) {
expectedReport := new(vuln.Report)
err := json.Unmarshal([]byte(rawReport), expectedReport)
require.NoError(suite.T(), err)
expectedReport.Vulnerabilities = nil
data, err := json.Marshal(expectedReport)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), string(data), summary)
}

19
src/pkg/scan/report/manager.go
View File

@ -104,13 +104,15 @@ const (
// basicManager is a default implementation of report manager.
type basicManager struct {
dao scan.DAO
dao scan.DAO
vulnDao scan.VulnerabilityRecordDao
}
// NewManager news basic manager.
func NewManager() Manager {
return &basicManager{
dao: scan.New(),
dao: scan.New(),
vulnDao: scan.NewVulnerabilityRecordDao(),
}
}
@ -136,6 +138,10 @@ func (bm *basicManager) Create(ctx context.Context, r *scan.Report) (string, err
}
func (bm *basicManager) Delete(ctx context.Context, uuid string) error {
_, err := bm.vulnDao.DeleteForReport(ctx, uuid)
if err != nil {
return err
}
query := q.Query{Keywords: q.KeyWords{"uuid": uuid}}
count, err := bm.dao.DeleteMany(ctx, query)
if err != nil {
@ -192,13 +198,20 @@ func (bm *basicManager) DeleteByDigests(ctx context.Context, digests ...string)
return nil
}
// delete the vulnerability records to the report UUID mapping for the digests
// provided
_, err := bm.vulnDao.DeleteForDigests(ctx, digests...)
if err != nil {
return err
}
var ol q.OrList
for _, digest := range digests {
ol.Values = append(ol.Values, digest)
}
query := q.Query{Keywords: q.KeyWords{"digest": &ol}}
_, err := bm.dao.DeleteMany(ctx, query)
_, err = bm.dao.DeleteMany(ctx, query)
return err
}

3
src/pkg/scan/report/report.go
View File

@ -25,7 +25,8 @@ type Merger func(r1, r2 interface{}) (interface{}, error)
// SupportedMergers declares mappings between mime type and report merger func.
var SupportedMergers = map[string]Merger{
v1.MimeTypeNativeReport: MergeNativeReport,
v1.MimeTypeNativeReport: MergeNativeReport,
v1.MimeTypeGenericVulnerabilityReport: MergeNativeReport,
}
// Merge merge report r1 and r2

6
src/pkg/scan/report/summary.go
View File

@ -55,7 +55,8 @@ type SummaryMerger func(s1, s2 interface{}) (interface{}, error)
// SupportedSummaryMergers declares mappings between mime type and summary merger func.
var SupportedSummaryMergers = map[string]SummaryMerger{
v1.MimeTypeNativeReport: MergeNativeSummary,
v1.MimeTypeNativeReport: MergeNativeSummary,
v1.MimeTypeGenericVulnerabilityReport: MergeNativeSummary,
}
// MergeSummary merge summary s1 and s2
@ -85,7 +86,8 @@ func MergeNativeSummary(s1, s2 interface{}) (interface{}, error) {
// SupportedGenerators declares mappings between mime type and summary generator func.
var SupportedGenerators = map[string]SummaryGenerator{
v1.MimeTypeNativeReport: GenerateNativeSummary,
v1.MimeTypeNativeReport: GenerateNativeSummary,
v1.MimeTypeGenericVulnerabilityReport: GenerateNativeSummary,
}
// GenerateSummary is a helper function to generate report

13
src/pkg/scan/report/supported_mime_test.go
View File

@ -77,3 +77,16 @@ func (suite *SupportedMimesSuite) TestResolveData() {
return
})
}
// TestResolveDataForGenericMimeType tests the ResolveData.
func (suite *SupportedMimesSuite) TestResolveDataForGenericMimeType() {
obj, err := ResolveData(v1.MimeTypeGenericVulnerabilityReport, suite.mockData)
require.NoError(suite.T(), err)
require.NotNil(suite.T(), obj)
require.Condition(suite.T(), func() (success bool) {
rp, ok := obj.(*vuln.Report)
success = ok && rp != nil && rp.Severity == vuln.High
return
})
}

3
src/pkg/scan/report/supported_mimes.go
View File

@ -26,7 +26,8 @@ import (
// SupportedMimes indicates what mime types are supported to render at UI end.
var SupportedMimes = map[string]interface{}{
// The native report type
v1.MimeTypeNativeReport: (*vuln.Report)(nil),
v1.MimeTypeNativeReport: (*vuln.Report)(nil),
v1.MimeTypeGenericVulnerabilityReport: (*vuln.Report)(nil),
}
// ResolveData is a helper func to parse the JSON data with the given mime type.

1
src/pkg/scan/rest/v1/client_test.go
View File

@ -126,6 +126,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ProducesMimeTypes: []string{
MimeTypeNativeReport,
MimeTypeRawReport,
MimeTypeGenericVulnerabilityReport,
},
}},
Properties: ScannerProperties{

2
src/pkg/scan/rest/v1/spec.go
View File

@ -39,6 +39,8 @@ const (
MimeTypeScanRequest = "application/vnd.scanner.adapter.scan.request+json; version=1.0"
// MimeTypeScanResponse defines the mime type for scan response
MimeTypeScanResponse = "application/vnd.scanner.adapter.scan.response+json; version=1.0"
// MimeTypeGenericVulnerabilityReport defines the MIME type for the generic report with enhanced information
MimeTypeGenericVulnerabilityReport = "application/vnd.security.vulnerability.report; version=1.1"
apiPrefix = "/api/v1"
)

24
src/pkg/scan/vuln/report.go
View File

@ -109,4 +109,28 @@ type VulnerabilityItem struct {
// The artifact digest which the vulnerability belonged
// e.g: sha256@ee1d00c5250b5a886b09be2d5f9506add35dfb557f1ef37a7e4b8f0138f32956
ArtifactDigest string `json:"artifact_digest"`
// The CVSS3 and CVSS2 based scores and attack vector for the vulnerability item
CVSSDetails CVSS `json:"preferred_cvss"`
// A separated list of CWE Ids associated with this vulnerability
// e.g. CWE-465,CWE-124
CWEIds []string `json:"cwe_ids"`
// A collection of vendor specific attributes for the vulnerability item
// with each attribute represented as a key-value pair.
VendorAttributes map[string]interface{} `json:"vendor_attributes"`
}
// CVSS holds the score and attack vector for the vulnerability based on the CVSS3 and CVSS2 standards
type CVSS struct {
// The CVSS-3 score for the vulnerability
// e.g. 2.5
ScoreV3 *float64 `json:"score_v3"`
// The CVSS-3 score for the vulnerability
// e.g. 2.5
ScoreV2 *float64 `json:"score_v2"`
// The CVSS-3 attack vector.
// e.g. CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N
VectorV3 string `json:"vector_v3"`
// The CVSS-3 attack vector.
// e.g. AV:L/AC:M/Au:N/C:P/I:N/A:N
VectorV2 string `json:"vector_v2"`
}

2
src/pkg/scan/vuln/summary.go
View File

@ -22,7 +22,7 @@ import (
)
// NativeReportSummary is the default supported scan report summary model.
// Generated based on the report with v1.MimeTypeNativeReport mime type.
// Generated based on the report with v1.MimeTypeNativeReport or the v1.MimeTypeGenericVulnerabilityReport mime type.
type NativeReportSummary struct {
ReportID string `json:"report_id"`
ScanStatus string `json:"scan_status"`

13
src/server/middleware/vulnerable/vulnerable.go
View File

@ -92,7 +92,7 @@ func Middleware() func(http.Handler) http.Handler {
}
allowlist := proj.CVEAllowlist.CVESet()
summaries, err := scanController.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport}, report.WithCVEAllowlist(&allowlist))
summaries, err := scanController.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport}, report.WithCVEAllowlist(&allowlist))
if err != nil {
logger.Errorf("get vulnerability summary of the artifact %s@%s failed, error: %v", art.RepositoryName, art.Digest, err)
return err
@ -102,10 +102,13 @@ func Middleware() func(http.Handler) http.Handler {
rawSummary, ok := summaries[v1.MimeTypeNativeReport]
if !ok {
// No report yet?
msg := fmt.Sprintf(`current image without vulnerability scanning cannot be pulled due to configured policy in 'Prevent images with vulnerability severity of "%s" or higher from running.' `+
`To continue with pull, please contact your project administrator for help.`, projectSeverity)
return errors.New(nil).WithCode(errors.PROJECTPOLICYVIOLATION).WithMessage(msg)
rawSummary, ok = summaries[v1.MimeTypeGenericVulnerabilityReport]
if !ok {
// No report yet?
msg := fmt.Sprintf(`current image without vulnerability scanning cannot be pulled due to configured policy in 'Prevent images with vulnerability severity of "%s" or higher from running.' `+
`To continue with pull, please contact your project administrator for help.`, projectSeverity)
return errors.New(nil).WithCode(errors.PROJECTPOLICYVIOLATION).WithMessage(msg)
}
}
summary, ok := rawSummary.(*vuln.NativeReportSummary)

2
src/server/v2.0/handler/assembler/vul.go
View File

@ -72,7 +72,7 @@ func (assembler *VulAssembler) Assemble(ctx context.Context) error {
artifact.SetAdditionLink(vulnerabilitiesAddition, version)
if assembler.withScanOverview {
overview, err := assembler.scanCtl.GetSummary(ctx, &artifact.Artifact, []string{v1.MimeTypeNativeReport})
overview, err := assembler.scanCtl.GetSummary(ctx, &artifact.Artifact, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport})
if err != nil {
log.Warningf("get scan summary of artifact %s@%s failed, error:%v", artifact.RepositoryName, artifact.Digest, err)
} else if len(overview) > 0 {

2
src/server/v2.0/handler/util.go
View File

@ -55,7 +55,7 @@ func boolValue(v *bool) bool {
}
func resolveVulnerabilitiesAddition(ctx context.Context, artifact *artifact.Artifact) (*processor.Addition, error) {
reports, err := scan.DefaultController.GetReport(ctx, artifact, []string{v1.MimeTypeNativeReport})
reports, err := scan.DefaultController.GetReport(ctx, artifact, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport})
if err != nil {
return nil, err
}

22
src/testing/pkg/scan/postprocessors/report_converters.go Normal file
View File

@ -0,0 +1,22 @@
package postprocessors
import (
"context"
mock "github.com/stretchr/testify/mock"
)
// ScanReportV1ToV2Converter is an auto-generated mock type for converting native Harbor report in JSON
// to relational schema
type ScanReportV1ToV2Converter struct {
mock.Mock
}
// ToRelationalSchema is a mock implementation of the scan report conversion
func (_c *ScanReportV1ToV2Converter) ToRelationalSchema(ctx context.Context, reportUUID string, registrationUUID string, digest string, reportData string) (string, string, error) {
return "mockId", reportData, nil
}
// ToRelationalSchema is a mock implementation of the scan report conversion
func (_c *ScanReportV1ToV2Converter) FromRelationalSchema(ctx context.Context, reportUUID string, artifactDigest string, reportData string) (string, error) {
return "mockId", nil
}