From a890b28e1ea2879e9f36a251dfa5c66cf559cd78 Mon Sep 17 00:00:00 2001 From: prahaladdarkin Date: Fri, 25 Dec 2020 06:17:46 +0530 Subject: [PATCH] Store vulnerability data from scanner into a relational format (#13616) feat: Store vulnerability report from scanner into a relational format Convert vulnerability report JSON obtained from scanner into a relational format describe in:https://github.com/goharbor/community/pull/145 Signed-off-by: prahaladdarkin --- .gitignore | 3 +- .../postgresql/0050_2.2.0_schema.up.sql | 41 ++ .../event/handler/webhook/scan/scan.go | 4 +- src/controller/p2p/preheat/enforcer.go | 12 +- src/controller/p2p/preheat/enforcer_test.go | 5 +- src/controller/scan/base_controller.go | 28 +- src/controller/scan/base_controller_test.go | 17 +- src/controller/scan/callback_test.go | 16 +- src/controller/scanner/base_controller.go | 6 +- .../scanner/base_controller_test.go | 38 ++ src/pkg/scan/dao/scan/model.go | 63 +++ src/pkg/scan/dao/scan/report_test.go | 6 +- src/pkg/scan/dao/scan/vulnerability.go | 240 +++++++++ src/pkg/scan/dao/scan/vulnerability_test.go | 296 +++++++++++ src/pkg/scan/job_test.go | 4 +- .../scan/postprocessors/report_converters.go | 198 +++++++ .../postprocessors/report_converters_test.go | 496 ++++++++++++++++++ src/pkg/scan/report/manager.go | 19 +- src/pkg/scan/report/report.go | 3 +- src/pkg/scan/report/summary.go | 6 +- src/pkg/scan/report/supported_mime_test.go | 13 + src/pkg/scan/report/supported_mimes.go | 3 +- src/pkg/scan/rest/v1/client_test.go | 1 + src/pkg/scan/rest/v1/spec.go | 2 + src/pkg/scan/vuln/report.go | 24 + src/pkg/scan/vuln/summary.go | 2 +- .../middleware/vulnerable/vulnerable.go | 13 +- src/server/v2.0/handler/assembler/vul.go | 2 +- src/server/v2.0/handler/util.go | 2 +- .../scan/postprocessors/report_converters.go | 22 + 30 files changed, 1537 insertions(+), 48 deletions(-) create mode 100644 src/pkg/scan/dao/scan/vulnerability.go create mode 100644 src/pkg/scan/dao/scan/vulnerability_test.go create mode 100644 src/pkg/scan/postprocessors/report_converters.go create mode 100644 src/pkg/scan/postprocessors/report_converters_test.go create mode 100644 src/testing/pkg/scan/postprocessors/report_converters.go diff --git a/.gitignore b/.gitignore index 68f90b57e..5a842f507 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ -!/contrib/helm/harbor +!/contrib/helm/harbor +*.code-workspace make/harbor.yml make/docker-compose.yml make/common/config/* diff --git a/make/migrations/postgresql/0050_2.2.0_schema.up.sql b/make/migrations/postgresql/0050_2.2.0_schema.up.sql index ad6c1bffb..8f5fb11f3 100644 --- a/make/migrations/postgresql/0050_2.2.0_schema.up.sql +++ b/make/migrations/postgresql/0050_2.2.0_schema.up.sql @@ -472,3 +472,44 @@ BEGIN update properties set v = cast(duration_in_days as text) WHERE k = 'robot_token_duration'; END IF; END $$; + +/* +Common vulnerability reporting schema. +Github proposal link : https://github.com/goharbor/community/pull/145 +*/ + +-- -------------------------------------------------- +-- Table Structure for `main.VulnerabilityRecord` +-- -------------------------------------------------- +CREATE TABLE IF NOT EXISTS "vulnerability_record" ( + "id" serial NOT NULL PRIMARY KEY, + "cve_id" text NOT NULL DEFAULT '' , + "registration_uuid" text NOT NULL DEFAULT '', + "package" text NOT NULL DEFAULT '' , + "package_version" text NOT NULL DEFAULT '' , + "package_type" text NOT NULL DEFAULT '' , + "severity" text NOT NULL DEFAULT '' , + "fixed_version" text, + "urls" text, + "cvss_score_v3" double precision, + "cvss_score_v2" double precision, + "cvss_vector_v3" text, + "cvss_vector_v2" text, + "description" text, + "cwe_ids" text, + "vendor_attributes" json, + UNIQUE ("cve_id", "registration_uuid", "package", "package_version"), + CONSTRAINT fk_registration_uuid FOREIGN KEY(registration_uuid) REFERENCES scanner_registration(uuid) ON DELETE CASCADE +); + +-- -------------------------------------------------- +-- Table Structure for `main.ReportVulnerabilityRecord` +-- -------------------------------------------------- +CREATE TABLE IF NOT EXISTS "report_vulnerability_record" ( + "id" serial NOT NULL PRIMARY KEY, + "report_uuid" text NOT NULL DEFAULT '' , + "vuln_record_id" bigint NOT NULL DEFAULT 0 , + UNIQUE ("report_uuid", "vuln_record_id"), + CONSTRAINT fk_vuln_record_id FOREIGN KEY(vuln_record_id) REFERENCES vulnerability_record(id) ON DELETE CASCADE, + CONSTRAINT fk_report_uuid FOREIGN KEY(report_uuid) REFERENCES scan_report(uuid) ON DELETE CASCADE +); diff --git a/src/controller/event/handler/webhook/scan/scan.go b/src/controller/event/handler/webhook/scan/scan.go index 33e245872..5f5381994 100644 --- a/src/controller/event/handler/webhook/scan/scan.go +++ b/src/controller/event/handler/webhook/scan/scan.go @@ -131,7 +131,7 @@ func constructScanImagePayload(event *event.ScanImageEvent, project *models.Proj // If the report is still not ready in the total time, then failed at then for i := 0; i < 10; i++ { // First check in case it is ready - if re, err := scan.DefaultController.GetReport(ctx, art, []string{v1.MimeTypeNativeReport}); err == nil { + if re, err := scan.DefaultController.GetReport(ctx, art, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport}); err == nil { if len(re) > 0 && len(re[0].Report) > 0 { break } @@ -143,7 +143,7 @@ func constructScanImagePayload(event *event.ScanImageEvent, project *models.Proj } // Add scan overview - summaries, err := scan.DefaultController.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport}) + summaries, err := scan.DefaultController.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport}) if err != nil { return nil, errors.Wrap(err, "construct scan payload") } diff --git a/src/controller/p2p/preheat/enforcer.go b/src/controller/p2p/preheat/enforcer.go index 4e75b178d..db26bb5aa 100644 --- a/src/controller/p2p/preheat/enforcer.go +++ b/src/controller/p2p/preheat/enforcer.go @@ -477,7 +477,7 @@ func (de *defaultEnforcer) startTask(ctx context.Context, executionID int64, can // getVulnerabilitySev gets the severity code value for the given artifact with allowlist option set func (de *defaultEnforcer) getVulnerabilitySev(ctx context.Context, p *models.Project, art *artifact.Artifact) (uint, error) { al := p.CVEAllowlist.CVESet() - r, err := de.scanCtl.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport}, report.WithCVEAllowlist(&al)) + r, err := de.scanCtl.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport}, report.WithCVEAllowlist(&al)) if err != nil { if errors.IsNotFoundErr(err) { // no vulnerability report @@ -487,11 +487,17 @@ func (de *defaultEnforcer) getVulnerabilitySev(ctx context.Context, p *models.Pr return defaultSeverityCode, errors.Wrap(err, "get vulnerability severity") } - // Severity is based on the native report format. + // Severity is based on the native report format or the generic vulnerability report format. // In case no supported report format, treat as same to the no report scenario sum, ok := r[v1.MimeTypeNativeReport] if !ok { - return defaultSeverityCode, nil + // check if a report with MimeTypeGenericVulnerabilityReport is present. + // return the default severity code only if it does not exist + sum, ok = r[v1.MimeTypeGenericVulnerabilityReport] + if !ok { + return defaultSeverityCode, nil + } + } sm, ok := sum.(*vuln.NativeReportSummary) diff --git a/src/controller/p2p/preheat/enforcer_test.go b/src/controller/p2p/preheat/enforcer_test.go index 42c078c95..e9962cf1d 100644 --- a/src/controller/p2p/preheat/enforcer_test.go +++ b/src/controller/p2p/preheat/enforcer_test.go @@ -106,7 +106,7 @@ func (suite *EnforcerTestSuite) SetupSuite() { fakeScanCtl.On("GetSummary", context.TODO(), mock.AnythingOfType("*artifact.Artifact"), - []string{v1.MimeTypeNativeReport}, + []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport}, mock.AnythingOfType("report.Option"), ).Return(mockVulnerabilitySummary(), nil) @@ -310,5 +310,8 @@ func mockVulnerabilitySummary() map[string]interface{} { v1.MimeTypeNativeReport: &vuln.NativeReportSummary{ Severity: vuln.Low, }, + v1.MimeTypeGenericVulnerabilityReport: &vuln.NativeReportSummary{ + Severity: vuln.Low, + }, } } diff --git a/src/controller/scan/base_controller.go b/src/controller/scan/base_controller.go index 1d0142e54..146f38e2d 100644 --- a/src/controller/scan/base_controller.go +++ b/src/controller/scan/base_controller.go @@ -37,6 +37,7 @@ import ( sca "github.com/goharbor/harbor/src/pkg/scan" "github.com/goharbor/harbor/src/pkg/scan/dao/scan" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scan/postprocessors" "github.com/goharbor/harbor/src/pkg/scan/report" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" "github.com/goharbor/harbor/src/pkg/scan/vuln" @@ -86,6 +87,8 @@ type basicController struct { execMgr task.ExecutionManager taskMgr task.Manager + // Converter for V1 report to V2 report + reportConverter postprocessors.NativeScanReportConverter } // NewController news a scan API controller @@ -125,6 +128,8 @@ func NewController() Controller { execMgr: task.ExecMgr, taskMgr: task.Mgr, + // Get the scan V1 to V2 report converters + reportConverter: postprocessors.NewNativeToRelationalSchemaConverter(), } } @@ -397,8 +402,8 @@ func (bc *basicController) GetReport(ctx context.Context, artifact *ar.Artifact, mimes := make([]string, 0) mimes = append(mimes, mimeTypes...) if len(mimes) == 0 { - // Retrieve native as default - mimes = append(mimes, v1.MimeTypeNativeReport) + // Retrieve native and the new generic format as default + mimes = append(mimes, v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport) } // Get current scanner settings @@ -593,10 +598,19 @@ func (bc *basicController) UpdateReport(ctx context.Context, report *sca.CheckIn return errors.New("no report found to update data") } - if err := bc.manager.UpdateReportData(ctx, rpl[0].UUID, report.RawReport); err != nil { + log.Infof("Converting report ID %s to the new V2 schema", rpl[0].UUID) + _, reportData, err := bc.reportConverter.ToRelationalSchema(ctx, rpl[0].UUID, rpl[0].RegistrationUUID, rpl[0].Digest, report.RawReport) + if err != nil { + return errors.Wrapf(err, "Failed to convert vulnerability data to new schema for report UUID : %s", rpl[0].UUID) + } + // update the original report with the new summarized report with all vulnerability data removed. + // this is required since the top level layers relay on the vuln.Report struct that + // contains additional metadata within the report which if stored in the new columns within the scan_report table + // would be redundant + if err := bc.manager.UpdateReportData(ctx, rpl[0].UUID, reportData); err != nil { return errors.Wrap(err, "scan controller: handle job hook") } - + log.Infof("Converted report ID %s to the new V2 schema", rpl[0].UUID) return nil } @@ -605,7 +619,6 @@ func (bc *basicController) DeleteReports(ctx context.Context, digests ...string) if err := bc.manager.DeleteByDigests(ctx, digests...); err != nil { return errors.Wrap(err, "scan controller: delete reports") } - return nil } @@ -835,6 +848,11 @@ func (bc *basicController) assembleReports(ctx context.Context, reports ...*scan } else { report.Status = job.ErrorStatus.String() } + completeReport, err := bc.reportConverter.FromRelationalSchema(ctx, report.UUID, report.Digest, report.Report) + if err != nil { + return err + } + report.Report = completeReport } return nil diff --git a/src/controller/scan/base_controller_test.go b/src/controller/scan/base_controller_test.go index 86a4cb916..f292f4729 100644 --- a/src/controller/scan/base_controller_test.go +++ b/src/controller/scan/base_controller_test.go @@ -42,6 +42,7 @@ import ( scannertesting "github.com/goharbor/harbor/src/testing/controller/scanner" ormtesting "github.com/goharbor/harbor/src/testing/lib/orm" "github.com/goharbor/harbor/src/testing/mock" + postprocessorstesting "github.com/goharbor/harbor/src/testing/pkg/scan/postprocessors" reporttesting "github.com/goharbor/harbor/src/testing/pkg/scan/report" tasktesting "github.com/goharbor/harbor/src/testing/pkg/task" "github.com/stretchr/testify/assert" @@ -60,11 +61,12 @@ type ControllerTestSuite struct { artifact *artifact.Artifact rawReport string - execMgr *tasktesting.ExecutionManager - taskMgr *tasktesting.Manager - reportMgr *reporttesting.Manager - ar artifact.Controller - c Controller + execMgr *tasktesting.ExecutionManager + taskMgr *tasktesting.Manager + reportMgr *reporttesting.Manager + ar artifact.Controller + c Controller + reportConverter *postprocessorstesting.ScanReportV1ToV2Converter } // TestController is the entry point of ControllerTestSuite. @@ -277,8 +279,9 @@ func (suite *ControllerTestSuite) SetupSuite() { cloneCtx: func(ctx context.Context) context.Context { return ctx }, makeCtx: func() context.Context { return context.TODO() }, - execMgr: suite.execMgr, - taskMgr: suite.taskMgr, + execMgr: suite.execMgr, + taskMgr: suite.taskMgr, + reportConverter: &postprocessorstesting.ScanReportV1ToV2Converter{}, } } diff --git a/src/controller/scan/callback_test.go b/src/controller/scan/callback_test.go index c24414b92..edfb38307 100644 --- a/src/controller/scan/callback_test.go +++ b/src/controller/scan/callback_test.go @@ -18,8 +18,6 @@ import ( "context" "encoding/json" "fmt" - "testing" - "github.com/goharbor/harbor/src/controller/artifact" "github.com/goharbor/harbor/src/controller/robot" "github.com/goharbor/harbor/src/jobservice/job" @@ -30,9 +28,11 @@ import ( artifacttesting "github.com/goharbor/harbor/src/testing/controller/artifact" robottesting "github.com/goharbor/harbor/src/testing/controller/robot" "github.com/goharbor/harbor/src/testing/mock" + postprocessorstesting "github.com/goharbor/harbor/src/testing/pkg/scan/postprocessors" reporttesting "github.com/goharbor/harbor/src/testing/pkg/scan/report" tasktesting "github.com/goharbor/harbor/src/testing/pkg/task" "github.com/stretchr/testify/suite" + "testing" ) type CallbackTestSuite struct { @@ -53,6 +53,7 @@ type CallbackTestSuite struct { taskMgr *tasktesting.Manager originalTaskMgr task.Manager + reportConverter *postprocessorstesting.ScanReportV1ToV2Converter } func (suite *CallbackTestSuite) SetupSuite() { @@ -73,11 +74,14 @@ func (suite *CallbackTestSuite) SetupSuite() { task.Mgr = suite.taskMgr suite.originalScanCtl = DefaultController + suite.reportConverter = &postprocessorstesting.ScanReportV1ToV2Converter{} + suite.scanCtl = &basicController{ - makeCtx: context.TODO, - manager: suite.reportMgr, - execMgr: suite.execMgr, - taskMgr: suite.taskMgr, + makeCtx: context.TODO, + manager: suite.reportMgr, + execMgr: suite.execMgr, + taskMgr: suite.taskMgr, + reportConverter: suite.reportConverter, } DefaultController = suite.scanCtl } diff --git a/src/controller/scanner/base_controller.go b/src/controller/scanner/base_controller.go index dd08d791d..3ba3844b0 100644 --- a/src/controller/scanner/base_controller.go +++ b/src/controller/scanner/base_controller.go @@ -296,17 +296,17 @@ func (bc *basicController) Ping(registration *scanner.Registration) (*v1.Scanner return nil, errors.Errorf("missing %s in consumes_mime_types", v1.MimeTypeDockerArtifact) } - // v1.MimeTypeNativeReport is required + // either of v1.MimeTypeNativeReport OR v1.MimeTypeGenericVulnerabilityReport is required found = false for _, pm := range ca.ProducesMimeTypes { - if pm == v1.MimeTypeNativeReport { + if pm == v1.MimeTypeNativeReport || pm == v1.MimeTypeGenericVulnerabilityReport { found = true break } } if !found { - return nil, errors.Errorf("missing %s in produces_mime_types", v1.MimeTypeNativeReport) + return nil, errors.Errorf("missing %s or %s in produces_mime_types", v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport) } } diff --git a/src/controller/scanner/base_controller_test.go b/src/controller/scanner/base_controller_test.go index 118d64e99..9239e4be3 100644 --- a/src/controller/scanner/base_controller_test.go +++ b/src/controller/scanner/base_controller_test.go @@ -66,6 +66,7 @@ func (suite *ControllerTestSuite) SetupTest() { ProducesMimeTypes: []string{ v1.MimeTypeNativeReport, v1.MimeTypeRawReport, + v1.MimeTypeGenericVulnerabilityReport, }, }}, Properties: v1.ScannerProperties{ @@ -260,6 +261,43 @@ func (suite *ControllerTestSuite) TestPing() { suite.NotNil(meta) } +// TestPingWithGenericMimeType tests ping for scanners supporting MIME type MimeTypeGenericVulnerabilityReport +func (suite *ControllerTestSuite) TestPingWithGenericMimeType() { + m := &v1.ScannerAdapterMetadata{ + Scanner: &v1.Scanner{ + Name: "Trivy", + Vendor: "Harbor", + Version: "0.1.0", + }, + Capabilities: []*v1.ScannerCapability{{ + ConsumesMimeTypes: []string{ + v1.MimeTypeOCIArtifact, + v1.MimeTypeDockerArtifact, + }, + ProducesMimeTypes: []string{ + v1.MimeTypeGenericVulnerabilityReport, + v1.MimeTypeRawReport, + }, + }}, + Properties: v1.ScannerProperties{ + "extra": "testing", + }, + } + mc := &v1testing.Client{} + mc.On("GetMetadata").Return(m, nil) + + mcp := &v1testing.ClientPool{} + mocktesting.OnAnything(mcp, "Get").Return(mc, nil) + suite.c = &basicController{ + manager: suite.mMgr, + proMetaMgr: suite.mMeta, + clientPool: mcp, + } + meta, err := suite.c.Ping(suite.sample) + require.NoError(suite.T(), err) + suite.NotNil(meta) +} + // TestGetMetadata ... func (suite *ControllerTestSuite) TestGetMetadata() { suite.sample.UUID = "uuid" diff --git a/src/pkg/scan/dao/scan/model.go b/src/pkg/scan/dao/scan/model.go index e0290e12e..0b483c2ab 100644 --- a/src/pkg/scan/dao/scan/model.go +++ b/src/pkg/scan/dao/scan/model.go @@ -43,3 +43,66 @@ func (r *Report) TableUnique() [][]string { {"digest", "registration_uuid", "mime_type"}, } } + +// VulnerabilityRecord of an individual vulnerability. Identifies an individual vulnerability item in the scan. +// Since multiple scanners could be registered with the projects, each scanner +// would have it's own definition for the same CVE ID. Hence a CVE ID is qualified along +// with the ID of the scanner that owns the CVE record definition. +// The scanner ID would be the same as the RegistrationUUID field of Report. +// Identified by the `cve_id` and `registration_uuid`. +// Relates to the image using the `digest` and to the report using the `report UUID` field +type VulnerabilityRecord struct { + ID int64 `orm:"pk;auto;column(id)"` + CVEID string `orm:"column(cve_id)"` + RegistrationUUID string `orm:"column(registration_uuid)"` + Package string `orm:"column(package)"` + PackageVersion string `orm:"column(package_version)"` + PackageType string `orm:"column(package_type)"` + Severity string `orm:"column(severity)"` + Fix string `orm:"column(fixed_version);null"` + URLs string `orm:"column(urls);null"` + CVE3Score *float64 `orm:"column(cvss_score_v3);null"` + CVE2Score *float64 `orm:"column(cvss_score_v2);null"` + CVSS3Vector string `orm:"column(cvss_vector_v3);null"` // e.g. CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N + CVSS2Vector string `orm:"column(cvss_vector_v2);null"` // e.g. AV:L/AC:M/Au:N/C:P/I:N/A:N + Description string `orm:"column(description);null"` + CWEIDs string `orm:"column(cwe_ids);null"` // e.g. CWE-476,CWE-123,CWE-234 + VendorAttributes string `orm:"column(vendor_attributes);type(json);null"` +} + +// ReportVulnerabilityRecord is relation table required to optimize data storage for both the +// vulnerability records and the scan report. +// identified by composite key (ID, Report) +// Since each scan report has a separate UUID, the composite key +// would ensure that the immutability of the historical scan reports is guaranteed. +// It is sufficient to store the int64 VulnerabilityRecord Id since the vulnerability records +// are uniquely identified in the table based on the ScannerID and the CVEID +type ReportVulnerabilityRecord struct { + ID int64 `orm:"pk;auto;column(id)"` + Report string `orm:"column(report_uuid);"` + VulnRecordID int64 `orm:"column(vuln_record_id);"` +} + +// TableName for VulnerabilityRecord +func (vr *VulnerabilityRecord) TableName() string { + return "vulnerability_record" +} + +// TableUnique for VulnerabilityRecord +func (vr *VulnerabilityRecord) TableUnique() [][]string { + return [][]string{ + {"cve_id", "registration_uuid", "package", "package_version"}, + } +} + +// TableName for ReportVulnerabilityRecord +func (rvr *ReportVulnerabilityRecord) TableName() string { + return "report_vulnerability_record" +} + +// TableUnique for ReportVulnerabilityRecord +func (rvr *ReportVulnerabilityRecord) TableUnique() [][]string { + return [][]string{ + {"report_uuid", "vuln_record_id"}, + } +} diff --git a/src/pkg/scan/dao/scan/report_test.go b/src/pkg/scan/dao/scan/report_test.go index 944f468ad..096c532b5 100644 --- a/src/pkg/scan/dao/scan/report_test.go +++ b/src/pkg/scan/dao/scan/report_test.go @@ -15,9 +15,9 @@ package scan import ( + htesting "github.com/goharbor/harbor/src/testing" "testing" - common "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/lib/orm" "github.com/goharbor/harbor/src/lib/q" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" @@ -27,7 +27,7 @@ import ( // ReportTestSuite is test suite of testing report DAO. type ReportTestSuite struct { - suite.Suite + htesting.Suite dao DAO } @@ -39,7 +39,7 @@ func TestReport(t *testing.T) { // SetupSuite prepares env for test suite. func (suite *ReportTestSuite) SetupSuite() { - common.PrepareTestForPostgresSQL() + suite.Suite.SetupSuite() suite.dao = New() } diff --git a/src/pkg/scan/dao/scan/vulnerability.go b/src/pkg/scan/dao/scan/vulnerability.go new file mode 100644 index 000000000..dba0ebfac --- /dev/null +++ b/src/pkg/scan/dao/scan/vulnerability.go @@ -0,0 +1,240 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "context" + "fmt" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" +) + +func init() { + orm.RegisterModel(new(VulnerabilityRecord), new(ReportVulnerabilityRecord)) +} + +// VulnerabilityRecordDao exposes the DAO layer contract to perform +// CRUD operations on vulnerability record objects +type VulnerabilityRecordDao interface { + // Create creates a new vulnerability record + Create(ctx context.Context, vr *VulnerabilityRecord) (int64, error) + // Delete deletes a vulnerability record + Delete(ctx context.Context, vr *VulnerabilityRecord) error + // List lists the vulnerability records + List(ctx context.Context, query *q.Query) ([]*VulnerabilityRecord, error) + // InsertForReport inserts vulnerability records for a report + InsertForReport(ctx context.Context, reportUUID string, vr *VulnerabilityRecord) (int64, error) + // GetForReport gets vulnerability records for a report + GetForReport(ctx context.Context, reportUUID string) ([]*VulnerabilityRecord, error) + // GetForScanner gets vulnerability records for a scanner + GetForScanner(ctx context.Context, registrationUUID string) ([]*VulnerabilityRecord, error) + // DeleteForScanner deletes vulnerability records for a scanner + DeleteForScanner(ctx context.Context, registrationUUID string) (int64, error) + // DeleteForReport deletes vulnerability records for a report + DeleteForReport(ctx context.Context, reportUUID string) (int64, error) + // DeleteForDigests deletes vulnerability records for a provided list of digests + DeleteForDigests(ctx context.Context, digests ...string) (int64, error) + // GetRecordIdsForScanner gets record ids of vulnerability records for a scanner + GetRecordIdsForScanner(ctx context.Context, registrationUUID string) ([]int, error) +} + +// NewVulnerabilityRecordDao returns a new dao to handle vulnerability data +func NewVulnerabilityRecordDao() VulnerabilityRecordDao { + return &vulnerabilityRecordDao{} +} + +type vulnerabilityRecordDao struct{} + +// Create creates new vulnerability record. +func (v *vulnerabilityRecordDao) Create(ctx context.Context, vr *VulnerabilityRecord) (int64, error) { + o, err := orm.FromContext(ctx) + var vrID int64 + err = orm.WithTransaction(func(ctx context.Context) error { + var err error + vrID, err = o.InsertOrUpdate(vr, "cve_id, registration_uuid, package, package_version") + return orm.WrapConflictError(err, "vulnerability already exists") + })(ctx) + if errors.IsConflictErr(err) { + if err := o.Read(vr, "cve_id", "registration_uuid", "package", "package_version"); err != nil { + return 0, err + } + return vr.ID, nil + } + return vrID, err +} + +// Delete deletes a vulnerability record +func (v *vulnerabilityRecordDao) Delete(ctx context.Context, vr *VulnerabilityRecord) error { + o, err := orm.FromContext(ctx) + if err != nil { + return err + } + _, err = o.Delete(vr, "CVEID", "RegistrationUUID", "Package", "PackageVersion") + + return err +} + +// List lists the vulnerability records with given query parameters. +// Keywords in query here will be enforced with `exact` way. +// If the registration ID (which = the scanner ID is not specified), the results +// would contain duplicate records for a CVE depending upon the number of registered +// scanners which individually store data about the CVE. In such cases, it is the +// responsibility of the calling code to de-duplicate the CVE records or bucket them +// per registered scanner +func (v *vulnerabilityRecordDao) List(ctx context.Context, query *q.Query) ([]*VulnerabilityRecord, error) { + o, err := orm.FromContext(ctx) + if err != nil { + return nil, err + } + qt := o.QueryTable(new(VulnerabilityRecord)) + + if query != nil { + if len(query.Keywords) > 0 { + for k, v := range query.Keywords { + if vv, ok := v.([]interface{}); ok { + qt = qt.Filter(fmt.Sprintf("%s__in", k), vv...) + continue + } + + qt = qt.Filter(k, v) + } + } + + if query.PageNumber > 0 && query.PageSize > 0 { + qt = qt.Limit(query.PageSize, (query.PageNumber-1)*query.PageSize) + } + } + + l := make([]*VulnerabilityRecord, 0) + _, err = qt.All(&l) + + return l, err +} + +// InsertForReport inserts a vulnerability record in the context of scan report +func (v *vulnerabilityRecordDao) InsertForReport(ctx context.Context, reportUUID string, vr *VulnerabilityRecord) (int64, error) { + + vrID, err := v.Create(ctx, vr) + + if err != nil { + return 0, err + } + + rvr := new(ReportVulnerabilityRecord) + rvr.Report = reportUUID + rvr.VulnRecordID = vrID + + o, err := orm.FromContext(ctx) + if err != nil { + return 0, err + } + _, rvrID, err := o.ReadOrCreate(rvr, "report_uuid", "vuln_record_id") + + return rvrID, err + +} + +// DeleteForReport deletes the vulnerability records for a single report +func (v *vulnerabilityRecordDao) DeleteForReport(ctx context.Context, reportUUID string) (int64, error) { + o, err := orm.FromContext(ctx) + if err != nil { + return 0, err + } + delCount, err := o.Delete(&ReportVulnerabilityRecord{Report: reportUUID}, "report_uuid") + return delCount, err +} + +// GetForReport gets all the vulnerability records for a report based on UUID +func (v *vulnerabilityRecordDao) GetForReport(ctx context.Context, reportUUID string) ([]*VulnerabilityRecord, error) { + vulnRecs := make([]*VulnerabilityRecord, 0) + o, err := orm.FromContext(ctx) + if err != nil { + return nil, err + } + query := `select vulnerability_record.* from vulnerability_record + inner join report_vulnerability_record on + vulnerability_record.id = report_vulnerability_record.vuln_record_id and report_vulnerability_record.report_uuid=?` + _, err = o.Raw(query, reportUUID).QueryRows(&vulnRecs) + return vulnRecs, err +} + +// GetForScanner gets all the vulnerability records known to a scanner +// identified by registrationUUID +func (v *vulnerabilityRecordDao) GetForScanner(ctx context.Context, registrationUUID string) ([]*VulnerabilityRecord, error) { + var vulnRecs []*VulnerabilityRecord + o, err := orm.FromContext(ctx) + if err != nil { + return nil, err + } + vulRec := new(VulnerabilityRecord) + qs := o.QueryTable(vulRec) + _, err = qs.Filter("registration_uuid", registrationUUID).All(&vulnRecs) + if err != nil { + return nil, err + } + return vulnRecs, nil +} + +// DeleteForScanner deletes all the vulnerability records for a given scanner +// identified by registrationUUID +func (v *vulnerabilityRecordDao) DeleteForScanner(ctx context.Context, registrationUUID string) (int64, error) { + o, err := orm.FromContext(ctx) + if err != nil { + return 0, err + } + vulnRec := new(VulnerabilityRecord) + vulnRec.RegistrationUUID = registrationUUID + return o.Delete(vulnRec, "registration_uuid") +} + +// DeleteForDigests deletes the report vulnerability record mappings for the provided +// set of digests +func (v *vulnerabilityRecordDao) DeleteForDigests(ctx context.Context, digests ...string) (int64, error) { + reportDao := New() + + ol := q.OrList{} + for _, digest := range digests { + ol.Values = append(ol.Values, digest) + } + reports, err := reportDao.List(ctx, &q.Query{Keywords: q.KeyWords{"digest": &ol}}) + if err != nil { + return 0, err + } + numRowsDeleted := int64(0) + for _, report := range reports { + delCount, err := v.DeleteForReport(ctx, report.UUID) + if err != nil { + return 0, err + } + numRowsDeleted += delCount + } + return numRowsDeleted, nil +} + +// GetRecordIdsForScanner retrieves the internal Ids of the vulnerability records for a given scanner +// identified by registrationUUID +func (v *vulnerabilityRecordDao) GetRecordIdsForScanner(ctx context.Context, registrationUUID string) ([]int, error) { + vulnRecordIds := make([]int, 0) + o, err := orm.FromContext(ctx) + if err != nil { + return nil, err + } + _, err = o.Raw("select id from vulnerability_record where registration_uuid = ?", registrationUUID).QueryRows(&vulnRecordIds) + if err != nil { + return vulnRecordIds, err + } + return vulnRecordIds, err +} diff --git a/src/pkg/scan/dao/scan/vulnerability_test.go b/src/pkg/scan/dao/scan/vulnerability_test.go new file mode 100644 index 000000000..22194258e --- /dev/null +++ b/src/pkg/scan/dao/scan/vulnerability_test.go @@ -0,0 +1,296 @@ +package scan + +import ( + "fmt" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + htesting "github.com/goharbor/harbor/src/testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "testing" +) + +const sampleReportWithCompleteVulnData = `{ + "generated_at": "2020-08-01T18:28:49.072885592Z", + "artifact": { + "repository": "library/ubuntu", + "digest": "sha256:d5b40885539615b9aeb7119516427959a158386af13e00d79a7da43ad1b3fb87", + "mime_type": "application/vnd.docker.distribution.manifest.v2+json" + }, + "scanner": { + "name": "Trivy", + "vendor": "Aqua Security", + "version": "v0.9.1" + }, + "severity": "Medium", + "vulnerabilities": [ + { + "id": "CVE-2019-18276", + "package": "bash", + "version": "5.0-6ubuntu1.1", + "severity": "Low", + "description": "An issue was discovered in disable_priv_mode in shell.c in GNU Bash through 5.0 patch 11. By default, if Bash is run with its effective UID not equal to its real UID, it will drop privileges by setting its effective UID to its real UID. However, it does so incorrectly. On Linux and other systems that support \"saved UID\" functionality, the saved UID is not dropped. An attacker with command execution in the shell can use \"enable -f\" for runtime loading of a new builtin, which can be a shared object that calls setuid() and therefore regains privileges. However, binaries running with an effective UID of 0 are unaffected.", + "links": [ + "http://packetstormsecurity.com/files/155498/Bash-5.0-Patch-11-Privilege-Escalation.html", + "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-18276", + "https://github.com/bminor/bash/commit/951bdaad7a18cc0dc1036bba86b18b90874d39ff", + "https://security.netapp.com/advisory/ntap-20200430-0003/", + "https://www.youtube.com/watch?v=-wGtxJ8opa8" + ], + "layer": { + "digest": "sha256:4739cd2f4f486596c583c79f6033f1a9dee019389d512603609494678c8ccd53", + "diff_id": "sha256:f66829086c450acd5f67d0529a58f7120926c890f04e17aa7f0e9365da86480a" + }, + "cwe_ids": ["CWE-476", "CWE-345"], + "cvss":{ + "score_v3": 3.2, + "score_v2": 2.3, + "vector_v3": "CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N", + "vector_v2": "AV:L/AC:M/Au:N/C:P/I:N/A:N" + }, + "vendor_attributes":[ { + "key": "foo", + "value": "bar" + }, + { + "key": "foo1", + "value": "bar1" + } + ] + } + ] +}` + +// VulnerabilityTestSuite is test suite of testing vulnerability DAO. +type VulnerabilityTestSuite struct { + htesting.Suite + rpUUID string + vulnerabilityRecordDao VulnerabilityRecordDao + dao DAO + registrationID string +} + +// TestVulnerabilityItem is the entry of ReportTestSuite. +func TestVulnerabilityItem(t *testing.T) { + suite.Run(t, &VulnerabilityTestSuite{}) +} + +// SetupSuite prepares env for test suite. +func (suite *VulnerabilityTestSuite) SetupSuite() { + suite.Suite.SetupSuite() + suite.rpUUID = "uuid" + suite.vulnerabilityRecordDao = NewVulnerabilityRecordDao() + suite.dao = New() + +} + +// SetupTest prepares env for test case. +func (suite *VulnerabilityTestSuite) SetupTest() { + r := &Report{ + UUID: "uuid", + Digest: "digest1001", + RegistrationUUID: "scannerId1", + MimeType: v1.MimeTypeNativeReport, + Status: job.PendingStatus.String(), + Report: sampleReportWithCompleteVulnData, + } + suite.registerScanner(r.RegistrationUUID) + suite.createReport(r) + vulns := generateVulnerabilityRecordsForReport("uuid", "scannerId1", 10) + for _, v := range vulns { + suite.insertVulnRecordForReport("uuid", v) + } + +} + +// TearDownTest clears enf for test case. +func (suite *VulnerabilityTestSuite) TearDownTest() { + registrations, err := scanner.ListRegistrations(&q.Query{}) + require.NoError(suite.T(), err, "Failed to cleanup scanner registrations") + for _, registration := range registrations { + err = scanner.DeleteRegistration(registration.UUID) + require.NoError(suite.T(), err, "Error when cleaning up scanner registrations") + } + reports, err := suite.dao.List(orm.Context(), &q.Query{}) + require.NoError(suite.T(), err) + for _, report := range reports { + suite.cleanUpAdditionalData(report.UUID, report.RegistrationUUID) + } +} + +// TestVulnerabilityRecordsListForReport tests listing of vulnerability record for reports +func (suite *VulnerabilityTestSuite) TestVulnerabilityRecordsListForReport() { + // create a second report and associate the same vulnerability record set to the report + r := &Report{ + UUID: "uuid1", + Digest: "digest1002", + RegistrationUUID: "scannerId2", + MimeType: v1.MimeTypeNativeReport, + Status: job.PendingStatus.String(), + Report: sampleReportWithCompleteVulnData, + } + suite.registerScanner(r.RegistrationUUID) + suite.createReport(r) + // insert a set of vulnerability records for this report. the vulnerability records + // belong to the same scanner + vulns := generateVulnerabilityRecordsForReport("uuid1", "scannerId2", 10) + for _, v := range vulns { + suite.insertVulnRecordForReport("uuid1", v) + } + + // fetch the records for the first report. Additionally assert that these records + // indeed belong to the same report being fetched and not to another report + { + vulns, err := suite.vulnerabilityRecordDao.GetForReport(orm.Context(), "uuid") + require.NoError(suite.T(), err, "Error when fetching vulnerability records for report") + require.True(suite.T(), len(vulns) > 0) + } + { + vulns, err := suite.vulnerabilityRecordDao.GetForReport(orm.Context(), "uuid1") + require.NoError(suite.T(), err, "Error when fetching vulnerability records for report") + require.True(suite.T(), len(vulns) > 0) + } + +} + +// TestGetVulnerabilityRecordsForScanner gets vulnerability records for scanner +func (suite *VulnerabilityTestSuite) TestGetVulnerabilityRecordsForScanner() { + + vulns, err := suite.vulnerabilityRecordDao.GetForScanner(orm.Context(), "scannerId1") + require.NoError(suite.T(), err, "Error when fetching vulnerability records for report") + require.True(suite.T(), len(vulns) > 0) +} + +// TestGetVulnerabilityRecordIdsForScanner gets vulnerability records for scanner +func (suite *VulnerabilityTestSuite) TestGetVulnerabilityRecordIdsForScanner() { + vulns, err := suite.vulnerabilityRecordDao.GetRecordIdsForScanner(orm.Context(), "scannerId1") + require.NoError(suite.T(), err, "Error when fetching vulnerability records for report") + require.True(suite.T(), len(vulns) > 0) +} + +// TestDeleteForDigest tests deleting vulnerability report for a specific digest +func (suite *VulnerabilityTestSuite) TestDeleteForDigest() { + // create a second report and associate the same vulnerability record set to the report + r := &Report{ + UUID: "uuid1", + Digest: "digest1", + RegistrationUUID: "scannerId2", + MimeType: v1.MimeTypeNativeReport, + Status: job.PendingStatus.String(), + Report: sampleReportWithCompleteVulnData, + } + suite.registerScanner(r.RegistrationUUID) + suite.createReport(r) + // insert a set of vulnerability records for this report. the vulnerability records + // belong to the same scanner + vulns := generateVulnerabilityRecordsForReport("uuid1", "scannerId2", 10) + for _, v := range vulns { + suite.insertVulnRecordForReport("uuid1", v) + } + delCount, err := suite.vulnerabilityRecordDao.DeleteForDigests(orm.Context(), "digest1") + require.NoError(suite.T(), err) + assert.Equal(suite.T(), int64(10), delCount) +} + +func (suite *VulnerabilityTestSuite) TestDuplicateRecords() { + r := &Report{ + UUID: "uuid2", + Digest: "digest1002", + RegistrationUUID: "scannerId1", + MimeType: v1.MimeTypeNativeReport, + Status: job.PendingStatus.String(), + Report: sampleReportWithCompleteVulnData, + } + suite.createReport(r) + vulns := generateVulnerabilityRecordsForReport("uuid2", "scannerId1", 10) + for _, v := range vulns { + suite.insertVulnRecordForReport("uuid2", v) + } +} + +// TestDeleteVulnerabilityRecord gets vulnerability records for scanner +func (suite *VulnerabilityTestSuite) TestDeleteVulnerabilityRecord() { + + vulns, err := suite.vulnerabilityRecordDao.GetForScanner(orm.Context(), "scannerId1") + require.NoError(suite.T(), err, "Error when fetching vulnerability records for report") + require.True(suite.T(), len(vulns) > 0) + for _, vuln := range vulns { + err = suite.vulnerabilityRecordDao.Delete(orm.Context(), vuln) + require.NoError(suite.T(), err) + } +} + +// TestListVulnerabilityRecord gets vulnerability records for scanner +func (suite *VulnerabilityTestSuite) TestListVulnerabilityRecord() { + + vulns, err := suite.vulnerabilityRecordDao.List(orm.Context(), &q.Query{Keywords: map[string]interface{}{"CVEID": "CVE-ID1"}}) + require.NoError(suite.T(), err, "Error when fetching vulnerability records for report") + require.True(suite.T(), len(vulns) > 0) +} + +func (suite *VulnerabilityTestSuite) createReport(r *Report) { + id, err := suite.dao.Create(orm.Context(), r) + require.NoError(suite.T(), err) + require.Condition(suite.T(), func() (success bool) { + success = id > 0 + return + }) +} + +func (suite *VulnerabilityTestSuite) insertVulnRecordForReport(reportUUID string, vr *VulnerabilityRecord) { + id, err := suite.vulnerabilityRecordDao.InsertForReport(orm.Context(), reportUUID, vr) + require.NoError(suite.T(), err) + require.True(suite.T(), id > 0, "Failed to insert vulnerability record row for report %s", reportUUID) +} + +func (suite *VulnerabilityTestSuite) cleanUpAdditionalData(reportID string, scannerID string) { + _, err := suite.dao.DeleteMany(orm.Context(), q.Query{Keywords: q.KeyWords{"uuid": reportID}}) + + require.NoError(suite.T(), err) + _, err = suite.vulnerabilityRecordDao.DeleteForReport(orm.Context(), reportID) + require.NoError(suite.T(), err, "Failed to cleanup records") + _, err = suite.vulnerabilityRecordDao.DeleteForScanner(orm.Context(), scannerID) + require.NoError(suite.T(), err, "Failed to delete vulnerability records") +} + +func (suite *VulnerabilityTestSuite) registerScanner(registrationUUID string) { + r := &scanner.Registration{ + UUID: registrationUUID, + Name: registrationUUID, + Description: "sample registration", + URL: fmt.Sprintf("https://sample.scanner.com/%s", registrationUUID), + } + + _, err := scanner.AddRegistration(r) + require.NoError(suite.T(), err, "add new registration") +} + +func generateVulnerabilityRecordsForReport(reportUUID string, registrationUUID string, numRecords int) []*VulnerabilityRecord { + vulns := make([]*VulnerabilityRecord, 0) + for i := 1; i <= numRecords; i++ { + vulnV2 := new(VulnerabilityRecord) + vulnV2.CVEID = fmt.Sprintf("CVE-ID%d", i) + vulnV2.Package = fmt.Sprintf("Package%d", i) + vulnV2.PackageVersion = "NotAvailable" + vulnV2.PackageType = "Unknown" + vulnV2.Fix = "1.0.0" + vulnV2.URLs = "url1" + vulnV2.RegistrationUUID = registrationUUID + if i%2 == 0 { + vulnV2.Severity = "High" + } else if i%3 == 0 { + vulnV2.Severity = "Medium" + } else if i%4 == 0 { + vulnV2.Severity = "Critical" + } else { + vulnV2.Severity = "Low" + } + vulns = append(vulns, vulnV2) + } + + return vulns +} diff --git a/src/pkg/scan/job_test.go b/src/pkg/scan/job_test.go index f746c2a57..4113b419c 100644 --- a/src/pkg/scan/job_test.go +++ b/src/pkg/scan/job_test.go @@ -103,7 +103,7 @@ func (suite *JobTestSuite) TestJob() { robotData, err := robot.ToJSON() require.NoError(suite.T(), err) - mimeTypes := []string{v1.MimeTypeNativeReport} + mimeTypes := []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport} jp := make(job.Parameters) jp[JobParamRegistration] = rData @@ -142,7 +142,7 @@ func (suite *JobTestSuite) TestJob() { jRep, err := json.Marshal(rp) require.NoError(suite.T(), err) - mc.On("GetScanReport", "scan_id", v1.MimeTypeNativeReport).Return(string(jRep), nil) + mc.On("GetScanReport", "scan_id", v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport).Return(string(jRep), nil) mocktesting.OnAnything(suite.mcp, "Get").Return(mc, nil) crp := &CheckInReport{ diff --git a/src/pkg/scan/postprocessors/report_converters.go b/src/pkg/scan/postprocessors/report_converters.go new file mode 100644 index 000000000..d566dfe6f --- /dev/null +++ b/src/pkg/scan/postprocessors/report_converters.go @@ -0,0 +1,198 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postprocessors + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/pkg/scan/dao/scan" + "github.com/goharbor/harbor/src/pkg/scan/vuln" +) + +// NativeScanReportConverter is an interface that establishes the contract for the conversion process of a harbor native vulnerability report +// It is the responsibility of the implementation to store the report in a manner easily retrievable using the +// report UUID +type NativeScanReportConverter interface { + ToRelationalSchema(ctx context.Context, reportUUID string, registrationUUID string, digest string, reportData string) (string, string, error) + FromRelationalSchema(ctx context.Context, reportUUID string, artifactDigest string, reportSummary string) (string, error) +} + +// nativeToRelationalSchemaConverter is responsible for converting the JSON scan report from the Harbor 1.0 format to +// the generic vulnerability format which follows a normalized storage schema. +type nativeToRelationalSchemaConverter struct { + dao scan.VulnerabilityRecordDao +} + +// NewNativeToRelationalSchemaConverter returns a new instance of a V1 report to V2 report converter +func NewNativeToRelationalSchemaConverter() NativeScanReportConverter { + return &nativeToRelationalSchemaConverter{dao: scan.NewVulnerabilityRecordDao()} +} + +// ToRelationalSchema converts the vulnerability report data present as JSON to the new relational VulnerabilityRecord instance +func (c *nativeToRelationalSchemaConverter) ToRelationalSchema(ctx context.Context, reportUUID string, registrationUUID string, digest string, reportData string) (string, string, error) { + + if len(reportData) == 0 { + log.Infof("There is no vulnerability report to toSchema for report UUID : %s", reportUUID) + return reportUUID, "", nil + } + + // parse the raw report with the V1 schema of the report to the normalized structures + rawReport := new(vuln.Report) + if err := json.Unmarshal([]byte(reportData), &rawReport); err != nil { + return "", "", errors.Wrap(err, fmt.Sprintf("Error when toSchema V1 report to V2")) + } + + if err := c.toSchema(ctx, reportUUID, registrationUUID, digest, reportData); err != nil { + return "", "", errors.Wrap(err, fmt.Sprintf("Error when converting vulnerability report")) + } + rawReport.Vulnerabilities = nil + data, err := json.Marshal(rawReport) + if err != nil { + return "", "", errors.Wrap(err, fmt.Sprintf("Error when persisting raw report summary")) + } + return reportUUID, string(data), nil + +} + +// FromRelationalSchema converts the generic vulnerability record stored in relational form to the +// native JSON blob. +func (c *nativeToRelationalSchemaConverter) FromRelationalSchema(ctx context.Context, reportUUID string, artifactDigest string, reportSummary string) (string, error) { + vulns, err := c.dao.GetForReport(ctx, reportUUID) + if err != nil { + return "", errors.Wrap(err, fmt.Sprintf("Error when toSchema generic vulnerability records for %s", reportUUID)) + } + rp, err := c.fromSchema(ctx, reportUUID, artifactDigest, reportSummary, vulns) + if err != nil { + return "", err + } + return rp, nil +} + +func (c *nativeToRelationalSchemaConverter) toSchema(ctx context.Context, reportUUID string, registrationUUID string, digest string, rawReportData string) error { + + var vulnReport vuln.Report + err := json.Unmarshal([]byte(rawReportData), &vulnReport) + if err != nil { + return err + } + for _, v := range vulnReport.Vulnerabilities { + vulnV2 := new(scan.VulnerabilityRecord) + vulnV2.CVEID = v.ID + vulnV2.Description = v.Description + vulnV2.Package = v.Package + vulnV2.PackageVersion = v.Version + vulnV2.PackageType = "Unknown" + vulnV2.Fix = v.FixVersion + vulnV2.URLs = strings.Join(v.Links, "|") + vulnV2.RegistrationUUID = registrationUUID + vulnV2.Severity = v.Severity.String() + + // process the CVSS scores if the data is available + if (vuln.CVSS{} != v.CVSSDetails) { + vulnV2.CVE3Score = v.CVSSDetails.ScoreV3 + vulnV2.CVE2Score = v.CVSSDetails.ScoreV2 + vulnV2.CVSS3Vector = v.CVSSDetails.VectorV3 + vulnV2.CVSS2Vector = v.CVSSDetails.VectorV2 + } + if len(v.CWEIds) > 0 { + vulnV2.CWEIDs = strings.Join(v.CWEIds, ",") + } + + // marshall the presented vendor attributes as a json string + if len(v.VendorAttributes) > 0 { + vendorAttributes, err := json.Marshal(v.VendorAttributes) + // set the vendor attributes iff unmarshalling is successful + if err == nil { + vulnV2.VendorAttributes = string(vendorAttributes) + } + } + + _, err = c.dao.InsertForReport(ctx, reportUUID, vulnV2) + if err != nil { + log.Warningf("Could not insert vulnerability record - report: %s, cve_id: %s, scanner: %s, package: %s, package_version: %s", reportUUID, v.ID, registrationUUID, v.Package, v.Version) + } + + } + log.Infof("Converted %d vulnerability records to the new schema for report ID %s and scanner Id %s", len(vulnReport.Vulnerabilities), reportUUID, registrationUUID) + return nil +} + +func (c *nativeToRelationalSchemaConverter) fromSchema(ctx context.Context, reportUUID string, artifactDigest string, reportSummary string, records []*scan.VulnerabilityRecord) (string, error) { + if len(reportSummary) == 0 { + return "", nil + } + vulnerabilityItems := make([]*vuln.VulnerabilityItem, 0) + for _, record := range records { + vi := new(vuln.VulnerabilityItem) + vi.ID = record.CVEID + vi.ArtifactDigest = artifactDigest + vi.CVSSDetails.ScoreV2 = record.CVE2Score + vi.CVSSDetails.ScoreV3 = record.CVE3Score + vi.CVSSDetails.VectorV2 = record.CVSS2Vector + vi.CVSSDetails.VectorV3 = record.CVSS3Vector + cweIDs := strings.Split(record.CWEIDs, ",") + for _, cweID := range cweIDs { + vi.CWEIds = append(vi.CWEIds, cweID) + } + vi.CWEIds = cweIDs + vi.Description = record.Description + vi.FixVersion = record.Fix + vi.Version = record.PackageVersion + urls := strings.Split(record.URLs, "|") + for _, url := range urls { + vi.Links = append(vi.Links, url) + } + vi.Severity = vuln.ParseSeverityVersion3(record.Severity) + vi.Package = record.Package + var vendorAttributes map[string]interface{} + _ = json.Unmarshal([]byte(record.VendorAttributes), &vendorAttributes) + vi.VendorAttributes = vendorAttributes + vulnerabilityItems = append(vulnerabilityItems, vi) + } + rp := new(vuln.Report) + err := json.Unmarshal([]byte(reportSummary), rp) + if err != nil { + return "", err + } + if len(vulnerabilityItems) > 0 { + rp.Vulnerabilities = make([]*vuln.VulnerabilityItem, 0) + for _, v := range vulnerabilityItems { + rp.Vulnerabilities = append(rp.Vulnerabilities, v) + } + } + + data, err := json.Marshal(rp) + if err != nil { + return "", err + } + return string(data), nil +} + +// GetNativeV1ReportFromResolvedData returns the native V1 scan report from the resolved +// interface data. +func (c *nativeToRelationalSchemaConverter) getNativeV1ReportFromResolvedData(ctx job.Context, rp interface{}) (*vuln.Report, error) { + report, ok := rp.(*vuln.Report) + if !ok { + return nil, errors.New("Data cannot be converted to v1 report format") + } + ctx.GetLogger().Infof("Converted raw data to report. Count of Vulnerabilities in report : %d", len(report.Vulnerabilities)) + return report, nil +} diff --git a/src/pkg/scan/postprocessors/report_converters_test.go b/src/pkg/scan/postprocessors/report_converters_test.go new file mode 100644 index 000000000..6d4e3d907 --- /dev/null +++ b/src/pkg/scan/postprocessors/report_converters_test.go @@ -0,0 +1,496 @@ +package postprocessors + +import ( + "encoding/json" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/scan/dao/scan" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scan/report" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/goharbor/harbor/src/pkg/scan/vuln" + htesting "github.com/goharbor/harbor/src/testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "testing" + "time" +) + +const sampleReport = `{ + "generated_at": "2020-08-01T18:28:49.072885592Z", + "artifact": { + "repository": "library/ubuntu", + "digest": "sha256:d5b40885539615b9aeb7119516427959a158386af13e00d79a7da43ad1b3fb87", + "mime_type": "application/vnd.docker.distribution.manifest.v2+json" + }, + "scanner": { + "name": "Trivy", + "vendor": "Aqua Security", + "version": "v0.9.1" + }, + "severity": "Medium", + "vulnerabilities": [ + { + "id": "CVE-2019-18276", + "package": "bash", + "version": "5.0-6ubuntu1.1", + "severity": "Low", + "description": "An issue was discovered in disable_priv_mode in shell.c in GNU Bash through 5.0 patch 11. By default, if Bash is run with its effective UID not equal to its real UID, it will drop privileges by setting its effective UID to its real UID. However, it does so incorrectly. On Linux and other systems that support \"saved UID\" functionality, the saved UID is not dropped. An attacker with command execution in the shell can use \"enable -f\" for runtime loading of a new builtin, which can be a shared object that calls setuid() and therefore regains privileges. However, binaries running with an effective UID of 0 are unaffected.", + "links": [ + "http://packetstormsecurity.com/files/155498/Bash-5.0-Patch-11-Privilege-Escalation.html", + "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-18276", + "https://github.com/bminor/bash/commit/951bdaad7a18cc0dc1036bba86b18b90874d39ff", + "https://security.netapp.com/advisory/ntap-20200430-0003/", + "https://www.youtube.com/watch?v=-wGtxJ8opa8" + ], + "layer": { + "digest": "sha256:4739cd2f4f486596c583c79f6033f1a9dee019389d512603609494678c8ccd53", + "diff_id": "sha256:f66829086c450acd5f67d0529a58f7120926c890f04e17aa7f0e9365da86480a" + } + } + ] +}` + +const sampleReportWithCWEAndCVSS = `{ + "generated_at": "2020-08-01T18:28:49.072885592Z", + "artifact": { + "repository": "library/ubuntu", + "digest": "sha256:d5b40885539615b9aeb7119516427959a158386af13e00d79a7da43ad1b3fb87", + "mime_type": "application/vnd.docker.distribution.manifest.v2+json" + }, + "scanner": { + "name": "Trivy", + "vendor": "Aqua Security", + "version": "v0.9.1" + }, + "severity": "Medium", + "vulnerabilities": [ + { + "id": "CVE-2019-18276", + "package": "bash", + "version": "5.0-6ubuntu1.1", + "severity": "Low", + "description": "An issue was discovered in disable_priv_mode in shell.c in GNU Bash through 5.0 patch 11. By default, if Bash is run with its effective UID not equal to its real UID, it will drop privileges by setting its effective UID to its real UID. However, it does so incorrectly. On Linux and other systems that support \"saved UID\" functionality, the saved UID is not dropped. An attacker with command execution in the shell can use \"enable -f\" for runtime loading of a new builtin, which can be a shared object that calls setuid() and therefore regains privileges. However, binaries running with an effective UID of 0 are unaffected.", + "links": [ + "http://packetstormsecurity.com/files/155498/Bash-5.0-Patch-11-Privilege-Escalation.html", + "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-18276", + "https://github.com/bminor/bash/commit/951bdaad7a18cc0dc1036bba86b18b90874d39ff", + "https://security.netapp.com/advisory/ntap-20200430-0003/", + "https://www.youtube.com/watch?v=-wGtxJ8opa8" + ], + "layer": { + "digest": "sha256:4739cd2f4f486596c583c79f6033f1a9dee019389d512603609494678c8ccd53", + "diff_id": "sha256:f66829086c450acd5f67d0529a58f7120926c890f04e17aa7f0e9365da86480a" + }, + "cwe_ids": ["CWE-476", "CWE-345"], + "preferred_cvss":{ + "score_v3": 3.2, + "score_v2": 2.3, + "vector_v3": "CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N", + "vector_v2": "AV:L/AC:M/Au:N/C:P/I:N/A:N" + } + } + ] +}` + +const sampleReportWithCompleteVulnData = `{ + "generated_at": "2020-08-01T18:28:49.072885592Z", + "artifact": { + "repository": "library/ubuntu", + "digest": "sha256:d5b40885539615b9aeb7119516427959a158386af13e00d79a7da43ad1b3fb87", + "mime_type": "application/vnd.docker.distribution.manifest.v2+json" + }, + "scanner": { + "name": "Trivy", + "vendor": "Aqua Security", + "version": "v0.9.1" + }, + "severity": "Medium", + "vulnerabilities": [ + { + "id": "CVE-2019-18276", + "package": "bash", + "version": "5.0-6ubuntu1.1", + "severity": "Low", + "description": "An issue was discovered in disable_priv_mode in shell.c in GNU Bash through 5.0 patch 11. By default, if Bash is run with its effective UID not equal to its real UID, it will drop privileges by setting its effective UID to its real UID. However, it does so incorrectly. On Linux and other systems that support \"saved UID\" functionality, the saved UID is not dropped. An attacker with command execution in the shell can use \"enable -f\" for runtime loading of a new builtin, which can be a shared object that calls setuid() and therefore regains privileges. However, binaries running with an effective UID of 0 are unaffected.", + "links": [ + "http://packetstormsecurity.com/files/155498/Bash-5.0-Patch-11-Privilege-Escalation.html", + "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-18276", + "https://github.com/bminor/bash/commit/951bdaad7a18cc0dc1036bba86b18b90874d39ff", + "https://security.netapp.com/advisory/ntap-20200430-0003/", + "https://www.youtube.com/watch?v=-wGtxJ8opa8" + ], + "layer": { + "digest": "sha256:4739cd2f4f486596c583c79f6033f1a9dee019389d512603609494678c8ccd53", + "diff_id": "sha256:f66829086c450acd5f67d0529a58f7120926c890f04e17aa7f0e9365da86480a" + }, + "cwe_ids": ["CWE-476", "CWE-345"], + "preferred_cvss":{ + "score_v3": 3.2, + "score_v2": 2.3, + "vector_v3": "CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N", + "vector_v2": "AV:L/AC:M/Au:N/C:P/I:N/A:N" + }, + "vendor_attributes":{ + "CVSS":{ + "nvd" : { + "V2Score": 7.1, + "V2Vector": "AV:L/AC:M/Au:N/C:P/I:N/A:N", + "V3Score": 6.5, + "V3Vector":"CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N" + } + } + } + } + ] +}` + +const sampleReportWithMixedSeverity = `{ + "generated_at": "2020-08-01T18:28:49.072885592Z", + "artifact": { + "repository": "library/ubuntu", + "digest": "sha256:d5b40885539615b9aeb7119516427959a158386af13e00d79a7da43ad1b3fb87", + "mime_type": "application/vnd.docker.distribution.manifest.v2+json" + }, + "scanner": { + "name": "Trivy", + "vendor": "Aqua Security", + "version": "v0.9.1" + }, + "severity": "Medium", + "vulnerabilities": [ + { + "id": "CVE-2019-18276", + "package": "bash", + "version": "5.0-6ubuntu1.1", + "severity": "Low", + "description": "An issue was discovered in disable_priv_mode in shell.c in GNU Bash through 5.0 patch 11. By default, if Bash is run with its effective UID not equal to its real UID, it will drop privileges by setting its effective UID to its real UID. However, it does so incorrectly. On Linux and other systems that support \"saved UID\" functionality, the saved UID is not dropped. An attacker with command execution in the shell can use \"enable -f\" for runtime loading of a new builtin, which can be a shared object that calls setuid() and therefore regains privileges. However, binaries running with an effective UID of 0 are unaffected.", + "links": [ + "http://packetstormsecurity.com/files/155498/Bash-5.0-Patch-11-Privilege-Escalation.html", + "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-18276", + "https://github.com/bminor/bash/commit/951bdaad7a18cc0dc1036bba86b18b90874d39ff", + "https://security.netapp.com/advisory/ntap-20200430-0003/", + "https://www.youtube.com/watch?v=-wGtxJ8opa8" + ], + "layer": { + "digest": "sha256:4739cd2f4f486596c583c79f6033f1a9dee019389d512603609494678c8ccd53", + "diff_id": "sha256:f66829086c450acd5f67d0529a58f7120926c890f04e17aa7f0e9365da86480a" + }, + "cwe_ids": ["CWE-476", "CWE-345"], + "preferred_cvss":{ + "score_v3": 3.2, + "score_v2": 2.3, + "vector_v3": "CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N", + "vector_v2": "AV:L/AC:M/Au:N/C:P/I:N/A:N" + }, + "vendor_attributes":{ + "CVSS":{ + "nvd" : { + "V2Score": 7.1, + "V2Vector": "AV:L/AC:M/Au:N/C:P/I:N/A:N", + "V3Score": 6.5, + "V3Vector":"CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N" + } + } + } + }, + { + "id": "CVE-2013-7445", + "package": "linux", + "version": "4.9.189-3+deb9u2", + "severity": "High", + "description": "An issue was discovered in disable_priv_mode in shell.c in GNU Bash through 5.0 patch 11. By default, if Bash is run with its effective UID not equal to its real UID, it will drop privileges by setting its effective UID to its real UID. However, it does so incorrectly. On Linux and other systems that support \"saved UID\" functionality, the saved UID is not dropped. An attacker with command execution in the shell can use \"enable -f\" for runtime loading of a new builtin, which can be a shared object that calls setuid() and therefore regains privileges. However, binaries running with an effective UID of 0 are unaffected.", + "links": [ + "http://packetstormsecurity.com/files/155498/Bash-5.0-Patch-11-Privilege-Escalation.html", + "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-18276", + "https://github.com/bminor/bash/commit/951bdaad7a18cc0dc1036bba86b18b90874d39ff", + "https://security.netapp.com/advisory/ntap-20200430-0003/", + "https://www.youtube.com/watch?v=-wGtxJ8opa8" + ], + "layer": { + "digest": "sha256:4739cd2f4f486596c583c79f6033f1a9dee019389d512603609494678c8ccd53", + "diff_id": "sha256:f66829086c450acd5f67d0529a58f7120926c890f04e17aa7f0e9365da86480a" + }, + "cwe_ids": ["CWE-476", "CWE-345"], + "preferred_cvss":{ + "score_v3": 3.2, + "score_v2": 2.3, + "vector_v3": "CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N", + "vector_v2": "AV:L/AC:M/Au:N/C:P/I:N/A:N" + }, + "vendor_attributes":{ + "CVSS":{ + "nvd" : { + "V2Score": 7.1, + "V2Vector": "AV:L/AC:M/Au:N/C:P/I:N/A:N", + "V3Score": 6.5, + "V3Vector":"CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N" + } + } + } + }, + { + "id": "CVE-2019-2182", + "package": "bash", + "version": "5.0-6ubuntu1.1", + "severity": "Medium", + "description": "An issue was discovered in disable_priv_mode in shell.c in GNU Bash through 5.0 patch 11. By default, if Bash is run with its effective UID not equal to its real UID, it will drop privileges by setting its effective UID to its real UID. However, it does so incorrectly. On Linux and other systems that support \"saved UID\" functionality, the saved UID is not dropped. An attacker with command execution in the shell can use \"enable -f\" for runtime loading of a new builtin, which can be a shared object that calls setuid() and therefore regains privileges. However, binaries running with an effective UID of 0 are unaffected.", + "links": [ + "http://packetstormsecurity.com/files/155498/Bash-5.0-Patch-11-Privilege-Escalation.html", + "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-18276", + "https://github.com/bminor/bash/commit/951bdaad7a18cc0dc1036bba86b18b90874d39ff", + "https://security.netapp.com/advisory/ntap-20200430-0003/", + "https://www.youtube.com/watch?v=-wGtxJ8opa8" + ], + "layer": { + "digest": "sha256:4739cd2f4f486596c583c79f6033f1a9dee019389d512603609494678c8ccd53", + "diff_id": "sha256:f66829086c450acd5f67d0529a58f7120926c890f04e17aa7f0e9365da86480a" + }, + "cwe_ids": ["CWE-476", "CWE-345"], + "preferred_cvss":{ + "score_v3": 3.2, + "score_v2": 2.3, + "vector_v3": "CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N", + "vector_v2": "AV:L/AC:M/Au:N/C:P/I:N/A:N" + }, + "vendor_attributes":{ + "CVSS":{ + "nvd" : { + "V2Score": 7.1, + "V2Vector": "AV:L/AC:M/Au:N/C:P/I:N/A:N", + "V3Score": 6.5, + "V3Vector":"CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N" + } + } + } + } + + ] +}` + +// TestReportConverterSuite +type TestReportConverterSuite struct { + htesting.Suite + rc NativeScanReportConverter + rpUUID string + vulnerabilityRecordDao scan.VulnerabilityRecordDao + reportDao scan.DAO + registrationID string +} + +// SetupTest prepares env for test cases. +func (suite *TestReportConverterSuite) SetupTest() { + suite.rpUUID = "reportUUID" + suite.registrationID = "ruuid" + r := &scanner.Registration{ + UUID: suite.registrationID, + Name: "forUT", + Description: "sample registration", + URL: "https://sample.scanner.com", + } + + _, err := scanner.AddRegistration(r) + require.NoError(suite.T(), err, "add new registration") +} + +// TestReportConverterTests specifies the test suite +func TestReportConverterTests(t *testing.T) { + suite.Run(t, &TestReportConverterSuite{}) +} + +// SetupSuite sets up the report converter suite test cases +func (suite *TestReportConverterSuite) SetupSuite() { + suite.rc = NewNativeToRelationalSchemaConverter() + suite.Suite.SetupSuite() + suite.vulnerabilityRecordDao = scan.NewVulnerabilityRecordDao() + suite.reportDao = scan.New() +} + +// TearDownTest clears test env for test cases. +func (suite *TestReportConverterSuite) TearDownTest() { + // No delete method defined in manager as no requirement, + // so, to clear env, call dao method here + scanner.DeleteRegistration(suite.registrationID) + reports, err := suite.reportDao.List(orm.Context(), &q.Query{}) + require.True(suite.T(), err == nil, "Failed to delete vulnerability records") + for _, report := range reports { + _, err := suite.reportDao.DeleteMany(orm.Context(), q.Query{Keywords: q.KeyWords{"uuid": report.UUID}}) + require.NoError(suite.T(), err) + _, err = suite.vulnerabilityRecordDao.DeleteForReport(orm.Context(), report.UUID) + require.NoError(suite.T(), err, "Failed to delete vulnerability records") + _, err = suite.vulnerabilityRecordDao.DeleteForScanner(orm.Context(), report.RegistrationUUID) + require.NoError(suite.T(), err, "Failed to delete vulnerability records") + } + +} + +// TestConvertReport tests the report conversion logic +func (suite *TestReportConverterSuite) TestConvertReport() { + rp := &scan.Report{ + Digest: "d1000", + RegistrationUUID: "ruuid", + MimeType: v1.MimeTypeNativeReport, + Report: sampleReport, + StartTime: time.Now(), + EndTime: time.Now().Add(1000), + UUID: "reportUUID", + } + suite.create(rp) + ruuid, summary, err := suite.rc.ToRelationalSchema(orm.Context(), rp.UUID, rp.RegistrationUUID, rp.Digest, rp.Report) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), rp.UUID, ruuid) + suite.validateReportSummary(summary, sampleReport) +} + +// TestConvertReportWithCWEAndCVSS tests the report conversion with CVSS and CWE information +func (suite *TestReportConverterSuite) TestConvertReportWithCWEAndCVSS() { + rp := &scan.Report{ + Digest: "d1000", + RegistrationUUID: "ruuid", + MimeType: v1.MimeTypeNativeReport, + Report: sampleReportWithCWEAndCVSS, + StartTime: time.Now(), + EndTime: time.Now().Add(1000), + UUID: "reportUUID1", + } + suite.create(rp) + ruuid, summary, err := suite.rc.ToRelationalSchema(orm.Context(), rp.UUID, rp.RegistrationUUID, rp.Digest, rp.Report) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), rp.UUID, ruuid) + suite.validateReportSummary(summary, sampleReportWithCWEAndCVSS) +} + +// TestConvertReportWithCompleteVulnData tests report conversion with complete vulnerability data +func (suite *TestReportConverterSuite) TestConvertReportWithCompleteVulnData() { + rp := &scan.Report{ + Digest: "d1000", + RegistrationUUID: "ruuid", + MimeType: v1.MimeTypeNativeReport, + Report: sampleReportWithCompleteVulnData, + StartTime: time.Now(), + EndTime: time.Now().Add(1000), + UUID: "reportUUID2", + } + suite.create(rp) + ruuid, summary, err := suite.rc.ToRelationalSchema(orm.Context(), rp.UUID, rp.RegistrationUUID, rp.Digest, rp.Report) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), rp.UUID, ruuid) + suite.validateReportSummary(summary, sampleReportWithCompleteVulnData) +} + +func (suite *TestReportConverterSuite) TestConvertToNativeReport() { + + rp := &scan.Report{ + Digest: "d1000", + RegistrationUUID: "ruuid", + MimeType: v1.MimeTypeNativeReport, + Report: sampleReportWithCompleteVulnData, + StartTime: time.Now(), + EndTime: time.Now().Add(1000), + UUID: "reportUUID2", + } + suite.create(rp) + _, summary, err := suite.rc.ToRelationalSchema(orm.Context(), rp.UUID, rp.RegistrationUUID, rp.Digest, sampleReportWithCompleteVulnData) + completeReport, err := suite.rc.FromRelationalSchema(orm.Context(), rp.UUID, rp.Digest, summary) + require.NoError(suite.T(), err) + v := new(vuln.Report) + err = json.Unmarshal([]byte(sampleReportWithCompleteVulnData), v) + require.NoError(suite.T(), err) + v.WithArtifactDigest(rp.Digest) + data, err := json.Marshal(v) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), string(data), completeReport) +} + +func (suite *TestReportConverterSuite) TestNativeReportSummaryAfterConversion() { + + rp := &scan.Report{ + Digest: "d1000", + RegistrationUUID: "ruuid", + MimeType: v1.MimeTypeGenericVulnerabilityReport, + Report: sampleReportWithMixedSeverity, + StartTime: time.Now(), + EndTime: time.Now().Add(1000), + UUID: "reportUUID2", + } + suite.create(rp) + _, summary, err := suite.rc.ToRelationalSchema(orm.Context(), rp.UUID, rp.RegistrationUUID, rp.Digest, rp.Report) + require.NoError(suite.T(), err) + completeReport, err := suite.rc.FromRelationalSchema(orm.Context(), rp.UUID, rp.Digest, summary) + require.NoError(suite.T(), err) + v := new(vuln.Report) + err = json.Unmarshal([]byte(rp.Report), v) + require.NoError(suite.T(), err) + v.WithArtifactDigest(rp.Digest) + data, err := json.Marshal(v) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), string(data), completeReport) + // validate that summarization happens correctly on this report + rp.Report = completeReport + rp.Status = job.SuccessStatus.String() + summ, err := report.GenerateSummary(rp) + require.NoError(suite.T(), err) + nativeReportSummary := summ.(*vuln.NativeReportSummary) + sevMapping := nativeReportSummary.Summary.Summary + assert.True(suite.T(), len(sevMapping) == 3, "Expected entries in severity mapping for 'High', 'Low', 'Medium'") + assert.Equal(suite.T(), 1, sevMapping[vuln.Low]) + assert.Equal(suite.T(), 1, sevMapping[vuln.High]) + assert.Equal(suite.T(), 1, sevMapping[vuln.Medium]) +} + +func (suite *TestReportConverterSuite) TestGenericVulnReportSummaryAfterConversion() { + + rp := &scan.Report{ + Digest: "d1000", + RegistrationUUID: "ruuid", + MimeType: v1.MimeTypeNativeReport, + Report: sampleReportWithMixedSeverity, + StartTime: time.Now(), + EndTime: time.Now().Add(1000), + UUID: "reportUUID2", + } + suite.create(rp) + _, summary, err := suite.rc.ToRelationalSchema(orm.Context(), rp.UUID, rp.RegistrationUUID, rp.Digest, rp.Report) + require.NoError(suite.T(), err) + completeReport, err := suite.rc.FromRelationalSchema(orm.Context(), rp.UUID, rp.Digest, summary) + require.NoError(suite.T(), err) + v := new(vuln.Report) + err = json.Unmarshal([]byte(rp.Report), v) + require.NoError(suite.T(), err) + v.WithArtifactDigest(rp.Digest) + data, err := json.Marshal(v) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), string(data), completeReport) + // validate that summarization happens correctly on this report + rp.Report = completeReport + rp.Status = job.SuccessStatus.String() + summ, err := report.GenerateSummary(rp) + require.NoError(suite.T(), err) + nativeReportSummary := summ.(*vuln.NativeReportSummary) + sevMapping := nativeReportSummary.Summary.Summary + assert.True(suite.T(), len(sevMapping) == 3, "Expected entries in severity mapping for 'High', 'Low', 'Medium'") + assert.Equal(suite.T(), 1, sevMapping[vuln.Low]) + assert.Equal(suite.T(), 1, sevMapping[vuln.High]) + assert.Equal(suite.T(), 1, sevMapping[vuln.Medium]) +} + +func (suite *TestReportConverterSuite) create(r *scan.Report) { + id, err := suite.reportDao.Create(orm.Context(), r) + require.NoError(suite.T(), err) + require.Condition(suite.T(), func() (success bool) { + success = id > 0 + return + }) +} + +func (suite *TestReportConverterSuite) validateReportSummary(summary string, rawReport string) { + expectedReport := new(vuln.Report) + err := json.Unmarshal([]byte(rawReport), expectedReport) + require.NoError(suite.T(), err) + expectedReport.Vulnerabilities = nil + data, err := json.Marshal(expectedReport) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), string(data), summary) +} diff --git a/src/pkg/scan/report/manager.go b/src/pkg/scan/report/manager.go index a002788e9..0bddb1898 100644 --- a/src/pkg/scan/report/manager.go +++ b/src/pkg/scan/report/manager.go @@ -104,13 +104,15 @@ const ( // basicManager is a default implementation of report manager. type basicManager struct { - dao scan.DAO + dao scan.DAO + vulnDao scan.VulnerabilityRecordDao } // NewManager news basic manager. func NewManager() Manager { return &basicManager{ - dao: scan.New(), + dao: scan.New(), + vulnDao: scan.NewVulnerabilityRecordDao(), } } @@ -136,6 +138,10 @@ func (bm *basicManager) Create(ctx context.Context, r *scan.Report) (string, err } func (bm *basicManager) Delete(ctx context.Context, uuid string) error { + _, err := bm.vulnDao.DeleteForReport(ctx, uuid) + if err != nil { + return err + } query := q.Query{Keywords: q.KeyWords{"uuid": uuid}} count, err := bm.dao.DeleteMany(ctx, query) if err != nil { @@ -192,13 +198,20 @@ func (bm *basicManager) DeleteByDigests(ctx context.Context, digests ...string) return nil } + // delete the vulnerability records to the report UUID mapping for the digests + // provided + _, err := bm.vulnDao.DeleteForDigests(ctx, digests...) + + if err != nil { + return err + } var ol q.OrList for _, digest := range digests { ol.Values = append(ol.Values, digest) } query := q.Query{Keywords: q.KeyWords{"digest": &ol}} - _, err := bm.dao.DeleteMany(ctx, query) + _, err = bm.dao.DeleteMany(ctx, query) return err } diff --git a/src/pkg/scan/report/report.go b/src/pkg/scan/report/report.go index 7e2c10c15..3b7379675 100644 --- a/src/pkg/scan/report/report.go +++ b/src/pkg/scan/report/report.go @@ -25,7 +25,8 @@ type Merger func(r1, r2 interface{}) (interface{}, error) // SupportedMergers declares mappings between mime type and report merger func. var SupportedMergers = map[string]Merger{ - v1.MimeTypeNativeReport: MergeNativeReport, + v1.MimeTypeNativeReport: MergeNativeReport, + v1.MimeTypeGenericVulnerabilityReport: MergeNativeReport, } // Merge merge report r1 and r2 diff --git a/src/pkg/scan/report/summary.go b/src/pkg/scan/report/summary.go index 373c3791b..083035f7e 100644 --- a/src/pkg/scan/report/summary.go +++ b/src/pkg/scan/report/summary.go @@ -55,7 +55,8 @@ type SummaryMerger func(s1, s2 interface{}) (interface{}, error) // SupportedSummaryMergers declares mappings between mime type and summary merger func. var SupportedSummaryMergers = map[string]SummaryMerger{ - v1.MimeTypeNativeReport: MergeNativeSummary, + v1.MimeTypeNativeReport: MergeNativeSummary, + v1.MimeTypeGenericVulnerabilityReport: MergeNativeSummary, } // MergeSummary merge summary s1 and s2 @@ -85,7 +86,8 @@ func MergeNativeSummary(s1, s2 interface{}) (interface{}, error) { // SupportedGenerators declares mappings between mime type and summary generator func. var SupportedGenerators = map[string]SummaryGenerator{ - v1.MimeTypeNativeReport: GenerateNativeSummary, + v1.MimeTypeNativeReport: GenerateNativeSummary, + v1.MimeTypeGenericVulnerabilityReport: GenerateNativeSummary, } // GenerateSummary is a helper function to generate report diff --git a/src/pkg/scan/report/supported_mime_test.go b/src/pkg/scan/report/supported_mime_test.go index 7b4602515..a7c46b156 100644 --- a/src/pkg/scan/report/supported_mime_test.go +++ b/src/pkg/scan/report/supported_mime_test.go @@ -77,3 +77,16 @@ func (suite *SupportedMimesSuite) TestResolveData() { return }) } + +// TestResolveDataForGenericMimeType tests the ResolveData. +func (suite *SupportedMimesSuite) TestResolveDataForGenericMimeType() { + obj, err := ResolveData(v1.MimeTypeGenericVulnerabilityReport, suite.mockData) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), obj) + require.Condition(suite.T(), func() (success bool) { + rp, ok := obj.(*vuln.Report) + success = ok && rp != nil && rp.Severity == vuln.High + + return + }) +} diff --git a/src/pkg/scan/report/supported_mimes.go b/src/pkg/scan/report/supported_mimes.go index 0d4d2184f..7ab8d13eb 100644 --- a/src/pkg/scan/report/supported_mimes.go +++ b/src/pkg/scan/report/supported_mimes.go @@ -26,7 +26,8 @@ import ( // SupportedMimes indicates what mime types are supported to render at UI end. var SupportedMimes = map[string]interface{}{ // The native report type - v1.MimeTypeNativeReport: (*vuln.Report)(nil), + v1.MimeTypeNativeReport: (*vuln.Report)(nil), + v1.MimeTypeGenericVulnerabilityReport: (*vuln.Report)(nil), } // ResolveData is a helper func to parse the JSON data with the given mime type. diff --git a/src/pkg/scan/rest/v1/client_test.go b/src/pkg/scan/rest/v1/client_test.go index 2a521e30a..e3ee8ae49 100644 --- a/src/pkg/scan/rest/v1/client_test.go +++ b/src/pkg/scan/rest/v1/client_test.go @@ -126,6 +126,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ProducesMimeTypes: []string{ MimeTypeNativeReport, MimeTypeRawReport, + MimeTypeGenericVulnerabilityReport, }, }}, Properties: ScannerProperties{ diff --git a/src/pkg/scan/rest/v1/spec.go b/src/pkg/scan/rest/v1/spec.go index cf7ff8647..bb46d1c1b 100644 --- a/src/pkg/scan/rest/v1/spec.go +++ b/src/pkg/scan/rest/v1/spec.go @@ -39,6 +39,8 @@ const ( MimeTypeScanRequest = "application/vnd.scanner.adapter.scan.request+json; version=1.0" // MimeTypeScanResponse defines the mime type for scan response MimeTypeScanResponse = "application/vnd.scanner.adapter.scan.response+json; version=1.0" + // MimeTypeGenericVulnerabilityReport defines the MIME type for the generic report with enhanced information + MimeTypeGenericVulnerabilityReport = "application/vnd.security.vulnerability.report; version=1.1" apiPrefix = "/api/v1" ) diff --git a/src/pkg/scan/vuln/report.go b/src/pkg/scan/vuln/report.go index c23d0db2b..fd8640bae 100644 --- a/src/pkg/scan/vuln/report.go +++ b/src/pkg/scan/vuln/report.go @@ -109,4 +109,28 @@ type VulnerabilityItem struct { // The artifact digest which the vulnerability belonged // e.g: sha256@ee1d00c5250b5a886b09be2d5f9506add35dfb557f1ef37a7e4b8f0138f32956 ArtifactDigest string `json:"artifact_digest"` + // The CVSS3 and CVSS2 based scores and attack vector for the vulnerability item + CVSSDetails CVSS `json:"preferred_cvss"` + // A separated list of CWE Ids associated with this vulnerability + // e.g. CWE-465,CWE-124 + CWEIds []string `json:"cwe_ids"` + // A collection of vendor specific attributes for the vulnerability item + // with each attribute represented as a key-value pair. + VendorAttributes map[string]interface{} `json:"vendor_attributes"` +} + +// CVSS holds the score and attack vector for the vulnerability based on the CVSS3 and CVSS2 standards +type CVSS struct { + // The CVSS-3 score for the vulnerability + // e.g. 2.5 + ScoreV3 *float64 `json:"score_v3"` + // The CVSS-3 score for the vulnerability + // e.g. 2.5 + ScoreV2 *float64 `json:"score_v2"` + // The CVSS-3 attack vector. + // e.g. CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N + VectorV3 string `json:"vector_v3"` + // The CVSS-3 attack vector. + // e.g. AV:L/AC:M/Au:N/C:P/I:N/A:N + VectorV2 string `json:"vector_v2"` } diff --git a/src/pkg/scan/vuln/summary.go b/src/pkg/scan/vuln/summary.go index decfbd34c..ff8341a15 100644 --- a/src/pkg/scan/vuln/summary.go +++ b/src/pkg/scan/vuln/summary.go @@ -22,7 +22,7 @@ import ( ) // NativeReportSummary is the default supported scan report summary model. -// Generated based on the report with v1.MimeTypeNativeReport mime type. +// Generated based on the report with v1.MimeTypeNativeReport or the v1.MimeTypeGenericVulnerabilityReport mime type. type NativeReportSummary struct { ReportID string `json:"report_id"` ScanStatus string `json:"scan_status"` diff --git a/src/server/middleware/vulnerable/vulnerable.go b/src/server/middleware/vulnerable/vulnerable.go index c616d2e8f..9a97bf5e3 100644 --- a/src/server/middleware/vulnerable/vulnerable.go +++ b/src/server/middleware/vulnerable/vulnerable.go @@ -92,7 +92,7 @@ func Middleware() func(http.Handler) http.Handler { } allowlist := proj.CVEAllowlist.CVESet() - summaries, err := scanController.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport}, report.WithCVEAllowlist(&allowlist)) + summaries, err := scanController.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport}, report.WithCVEAllowlist(&allowlist)) if err != nil { logger.Errorf("get vulnerability summary of the artifact %s@%s failed, error: %v", art.RepositoryName, art.Digest, err) return err @@ -102,10 +102,13 @@ func Middleware() func(http.Handler) http.Handler { rawSummary, ok := summaries[v1.MimeTypeNativeReport] if !ok { - // No report yet? - msg := fmt.Sprintf(`current image without vulnerability scanning cannot be pulled due to configured policy in 'Prevent images with vulnerability severity of "%s" or higher from running.' `+ - `To continue with pull, please contact your project administrator for help.`, projectSeverity) - return errors.New(nil).WithCode(errors.PROJECTPOLICYVIOLATION).WithMessage(msg) + rawSummary, ok = summaries[v1.MimeTypeGenericVulnerabilityReport] + if !ok { + // No report yet? + msg := fmt.Sprintf(`current image without vulnerability scanning cannot be pulled due to configured policy in 'Prevent images with vulnerability severity of "%s" or higher from running.' `+ + `To continue with pull, please contact your project administrator for help.`, projectSeverity) + return errors.New(nil).WithCode(errors.PROJECTPOLICYVIOLATION).WithMessage(msg) + } } summary, ok := rawSummary.(*vuln.NativeReportSummary) diff --git a/src/server/v2.0/handler/assembler/vul.go b/src/server/v2.0/handler/assembler/vul.go index 3e7ebadc2..08a9fbf78 100644 --- a/src/server/v2.0/handler/assembler/vul.go +++ b/src/server/v2.0/handler/assembler/vul.go @@ -72,7 +72,7 @@ func (assembler *VulAssembler) Assemble(ctx context.Context) error { artifact.SetAdditionLink(vulnerabilitiesAddition, version) if assembler.withScanOverview { - overview, err := assembler.scanCtl.GetSummary(ctx, &artifact.Artifact, []string{v1.MimeTypeNativeReport}) + overview, err := assembler.scanCtl.GetSummary(ctx, &artifact.Artifact, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport}) if err != nil { log.Warningf("get scan summary of artifact %s@%s failed, error:%v", artifact.RepositoryName, artifact.Digest, err) } else if len(overview) > 0 { diff --git a/src/server/v2.0/handler/util.go b/src/server/v2.0/handler/util.go index d2ed4bc5e..0f4e80fc3 100644 --- a/src/server/v2.0/handler/util.go +++ b/src/server/v2.0/handler/util.go @@ -55,7 +55,7 @@ func boolValue(v *bool) bool { } func resolveVulnerabilitiesAddition(ctx context.Context, artifact *artifact.Artifact) (*processor.Addition, error) { - reports, err := scan.DefaultController.GetReport(ctx, artifact, []string{v1.MimeTypeNativeReport}) + reports, err := scan.DefaultController.GetReport(ctx, artifact, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport}) if err != nil { return nil, err } diff --git a/src/testing/pkg/scan/postprocessors/report_converters.go b/src/testing/pkg/scan/postprocessors/report_converters.go new file mode 100644 index 000000000..dfc750241 --- /dev/null +++ b/src/testing/pkg/scan/postprocessors/report_converters.go @@ -0,0 +1,22 @@ +package postprocessors + +import ( + "context" + mock "github.com/stretchr/testify/mock" +) + +// ScanReportV1ToV2Converter is an auto-generated mock type for converting native Harbor report in JSON +// to relational schema +type ScanReportV1ToV2Converter struct { + mock.Mock +} + +// ToRelationalSchema is a mock implementation of the scan report conversion +func (_c *ScanReportV1ToV2Converter) ToRelationalSchema(ctx context.Context, reportUUID string, registrationUUID string, digest string, reportData string) (string, string, error) { + return "mockId", reportData, nil +} + +// ToRelationalSchema is a mock implementation of the scan report conversion +func (_c *ScanReportV1ToV2Converter) FromRelationalSchema(ctx context.Context, reportUUID string, artifactDigest string, reportData string) (string, error) { + return "mockId", nil +}