mirror of https://github.com/goharbor/harbor.git
Update table scan_report and extract cvss_v3_score from vendor attribute (#18854)
For better performance when query cve information, add summary information to scan_report
Extract cve_score from vendor attribute in vulnerability_record
SQL migrate script for the update
Signed-off-by: stonezdj <daojunz@vmware.com>
This commit is contained in:
stonezdj(Daojun Zhang)
GitHub
parent
7435c8c5ab
commit
d84b1d07d2
|
|
@ -4,4 +4,73 @@ CREATE INDEX IF NOT EXISTS idx_task_extra_attrs_report_uuids ON task USING gin (
|
|||
UPDATE execution SET vendor_id = (extra_attrs -> 'artifact' ->> 'id')::integer
|
||||
WHERE jsonb_path_exists(extra_attrs::jsonb, '$.artifact.id')
|
||||
AND vendor_id IN (SELECT id FROM scanner_registration)
|
||||
AND vendor_type = 'IMAGE_SCAN';
|
||||
AND vendor_type = 'IMAGE_SCAN';
|
||||
|
||||
/* extract score from vendor attribute */
|
||||
UPDATE vulnerability_record
|
||||
SET cvss_score_v3 = (vendor_attributes->'CVSS'->'nvd'->>'V3Score')::double precision
|
||||
WHERE jsonb_path_exists(vendor_attributes::jsonb, '$.CVSS.nvd.V3Score');
|
||||
|
||||
/* add summary information in scan_report */
|
||||
ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS critical_cnt BIGINT;
|
||||
ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS high_cnt BIGINT;
|
||||
ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS medium_cnt BIGINT;
|
||||
ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS low_cnt BIGINT;
|
||||
ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS none_cnt BIGINT;
|
||||
ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS unknown_cnt BIGINT;
|
||||
ALTER TABLE scan_report ADD COLUMN IF NOT EXISTS fixable_cnt BIGINT;
|
||||
|
||||
/* extract summary information for previous scan_report */
|
||||
DO
|
||||
$$
|
||||
DECLARE
|
||||
report RECORD;
|
||||
v RECORD;
|
||||
critical_count BIGINT;
|
||||
high_count BIGINT;
|
||||
none_count BIGINT;
|
||||
medium_count BIGINT;
|
||||
low_count BIGINT;
|
||||
unknown_count BIGINT;
|
||||
fixable_count BIGINT;
|
||||
BEGIN
|
||||
FOR report IN SELECT uuid FROM scan_report
|
||||
LOOP
|
||||
critical_count := 0;
|
||||
high_count := 0;
|
||||
medium_count := 0;
|
||||
none_count := 0;
|
||||
low_count := 0;
|
||||
unknown_count := 0;
|
||||
FOR v IN SELECT vr.severity, vr.fixed_version
|
||||
FROM report_vulnerability_record rvr,
|
||||
vulnerability_record vr
|
||||
WHERE rvr.report_uuid = report.uuid
|
||||
AND rvr.vuln_record_id = vr.id
|
||||
LOOP
|
||||
IF v.severity = 'Critical' THEN
|
||||
critical_count = critical_count + 1;
|
||||
ELSIF v.severity = 'High' THEN
|
||||
high_count = high_count + 1;
|
||||
ELSIF v.severity = 'Medium' THEN
|
||||
medium_count = medium_count + 1;
|
||||
ELSIF v.severity = 'Low' THEN
|
||||
low_count = low_count + 1;
|
||||
ELSIF v.severity = 'None' THEN
|
||||
none_count = none_count + 1;
|
||||
ELSIF v.severity = 'Unknown' THEN
|
||||
unknown_count = unknown_count + 1;
|
||||
ELSIF v.fixed_version IS NOT NULL THEN
|
||||
fixable_count = fixable_count + 1;
|
||||
END IF;
|
||||
END LOOP;
|
||||
UPDATE scan_report
|
||||
SET critical_cnt = critical_count,
|
||||
high_cnt = high_count,
|
||||
medium_cnt = medium_count,
|
||||
low_cnt = low_count,
|
||||
unknown_cnt = unknown_count
|
||||
WHERE uuid = report.uuid;
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
|
|
|
|||
|
|
@ -22,16 +22,22 @@ import (
|
|||
// Report of the scan.
|
||||
// Identified by the `digest`, `registration_uuid` and `mime_type`.
|
||||
type Report struct {
|
||||
ID int64 `orm:"pk;auto;column(id)"`
|
||||
UUID string `orm:"unique;column(uuid)"`
|
||||
Digest string `orm:"column(digest)"`
|
||||
RegistrationUUID string `orm:"column(registration_uuid)"`
|
||||
MimeType string `orm:"column(mime_type)"`
|
||||
Report string `orm:"column(report);type(json)"`
|
||||
|
||||
Status string `orm:"-"`
|
||||
StartTime time.Time `orm:"-"`
|
||||
EndTime time.Time `orm:"-"`
|
||||
ID int64 `orm:"pk;auto;column(id)"`
|
||||
UUID string `orm:"unique;column(uuid)"`
|
||||
Digest string `orm:"column(digest)"`
|
||||
RegistrationUUID string `orm:"column(registration_uuid)"`
|
||||
MimeType string `orm:"column(mime_type)"`
|
||||
Report string `orm:"column(report);type(json)"`
|
||||
CriticalCnt int64 `orm:"column(critical_cnt)"`
|
||||
HighCnt int64 `orm:"column(high_cnt)"`
|
||||
MediumCnt int64 `orm:"column(medium_cnt)"`
|
||||
LowCnt int64 `orm:"column(low_cnt)"`
|
||||
UnknownCnt int64 `orm:"column(unknown_cnt)"`
|
||||
NoneCnt int64 `orm:"column(none_cnt)"`
|
||||
FixableCnt int64 `orm:"column(fixable_cnt)"`
|
||||
Status string `orm:"-"`
|
||||
StartTime time.Time `orm:"-"`
|
||||
EndTime time.Time `orm:"-"`
|
||||
}
|
||||
|
||||
// TableName for Report
|
||||
|
|
|
|||
|
|
@ -36,6 +36,8 @@ type DAO interface {
|
|||
List(ctx context.Context, query *q.Query) ([]*Report, error)
|
||||
// UpdateReportData only updates the `report` column with conditions matched.
|
||||
UpdateReportData(ctx context.Context, uuid string, report string) error
|
||||
// Update update report
|
||||
Update(ctx context.Context, r *Report, cols ...string) error
|
||||
}
|
||||
|
||||
// New returns an instance of the default DAO
|
||||
|
|
@ -97,3 +99,14 @@ func (d *dao) UpdateReportData(ctx context.Context, uuid string, report string)
|
|||
_, err = qt.Filter("uuid", uuid).Update(data)
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *dao) Update(ctx context.Context, r *Report, cols ...string) error {
|
||||
o, err := orm.FromContext(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := o.Update(r, cols...); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import (
|
|||
"github.com/goharbor/harbor/src/lib/log"
|
||||
"github.com/goharbor/harbor/src/lib/q"
|
||||
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
|
||||
"github.com/goharbor/harbor/src/pkg/scan/report"
|
||||
"github.com/goharbor/harbor/src/pkg/scan/vuln"
|
||||
)
|
||||
|
||||
|
|
@ -151,7 +152,7 @@ func (c *nativeToRelationalSchemaConverter) toSchema(ctx context.Context, report
|
|||
var newRecords []*scan.VulnerabilityRecord
|
||||
for _, v := range vulnReport.Vulnerabilities {
|
||||
if !s.Exists(v.Key()) {
|
||||
newRecords = append(newRecords, toVulnerabilityRecord(v, registrationUUID))
|
||||
newRecords = append(newRecords, toVulnerabilityRecord(ctx, v, registrationUUID))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -230,7 +231,7 @@ func (c *nativeToRelationalSchemaConverter) getNativeV1ReportFromResolvedData(ct
|
|||
return report, nil
|
||||
}
|
||||
|
||||
func toVulnerabilityRecord(item *vuln.VulnerabilityItem, registrationUUID string) *scan.VulnerabilityRecord {
|
||||
func toVulnerabilityRecord(ctx context.Context, item *vuln.VulnerabilityItem, registrationUUID string) *scan.VulnerabilityRecord {
|
||||
record := new(scan.VulnerabilityRecord)
|
||||
|
||||
record.CVEID = item.ID
|
||||
|
|
@ -261,6 +262,12 @@ func toVulnerabilityRecord(item *vuln.VulnerabilityItem, registrationUUID string
|
|||
if err == nil {
|
||||
record.VendorAttributes = string(vendorAttributes)
|
||||
}
|
||||
|
||||
// parse the NVD score from the vendor attributes
|
||||
nvdScore := parseScoreFromVendorAttribute(ctx, string(vendorAttributes))
|
||||
if record.CVE3Score == nil {
|
||||
record.CVE3Score = &nvdScore
|
||||
}
|
||||
}
|
||||
|
||||
return record
|
||||
|
|
@ -290,3 +297,78 @@ func toVulnerabilityItem(record *scan.VulnerabilityRecord, artifactDigest string
|
|||
|
||||
return item
|
||||
}
|
||||
|
||||
// updateReport updates the report summary with the vulnerability counts
|
||||
func (c *nativeToRelationalSchemaConverter) updateReport(ctx context.Context, vulnerabilities []*vuln.VulnerabilityItem, reportUUID string) error {
|
||||
log.G(ctx).WithFields(log.Fields{"reportUUID": reportUUID}).Debugf("Update report summary for report")
|
||||
CriticalCnt := int64(0)
|
||||
HighCnt := int64(0)
|
||||
MediumCnt := int64(0)
|
||||
LowCnt := int64(0)
|
||||
NoneCnt := int64(0)
|
||||
UnknownCnt := int64(0)
|
||||
FixableCnt := int64(0)
|
||||
|
||||
for _, v := range vulnerabilities {
|
||||
v.Severity = vuln.ParseSeverityVersion3(v.Severity.String())
|
||||
switch v.Severity {
|
||||
case vuln.Critical:
|
||||
CriticalCnt++
|
||||
case vuln.High:
|
||||
HighCnt++
|
||||
case vuln.Medium:
|
||||
MediumCnt++
|
||||
case vuln.Low:
|
||||
LowCnt++
|
||||
case vuln.None:
|
||||
NoneCnt++
|
||||
case vuln.Unknown:
|
||||
UnknownCnt++
|
||||
}
|
||||
if len(v.FixVersion) > 0 {
|
||||
FixableCnt++
|
||||
}
|
||||
}
|
||||
|
||||
reports, err := report.Mgr.List(ctx, q.New(q.KeyWords{"uuid": reportUUID}))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(reports) == 0 {
|
||||
return errors.New(nil).WithMessage("report not found, uuid:%v", reportUUID)
|
||||
}
|
||||
r := reports[0]
|
||||
|
||||
r.CriticalCnt = CriticalCnt
|
||||
r.HighCnt = HighCnt
|
||||
r.MediumCnt = MediumCnt
|
||||
r.LowCnt = LowCnt
|
||||
r.NoneCnt = NoneCnt
|
||||
r.FixableCnt = FixableCnt
|
||||
r.UnknownCnt = UnknownCnt
|
||||
|
||||
return report.Mgr.Update(ctx, r, "CriticalCnt", "HighCnt", "MediumCnt", "LowCnt", "NoneCnt", "UnknownCnt", "FixableCnt")
|
||||
}
|
||||
|
||||
// CVSS ...
|
||||
type CVSS struct {
|
||||
NVD Nvd `json:"nvd"`
|
||||
}
|
||||
|
||||
// Nvd ...
|
||||
type Nvd struct {
|
||||
V3Score float64 `json:"V3Score"`
|
||||
}
|
||||
|
||||
func parseScoreFromVendorAttribute(ctx context.Context, vendorAttribute string) (NvdV3Score float64) {
|
||||
var data map[string]CVSS
|
||||
err := json.Unmarshal([]byte(vendorAttribute), &data)
|
||||
if err != nil {
|
||||
log.G(ctx).Errorf("failed to parse vendor_attribute, error %v", err)
|
||||
return 0
|
||||
}
|
||||
if cvss, ok := data["CVSS"]; ok {
|
||||
return cvss.NVD.V3Score
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
package postprocessors
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
|
@ -294,6 +295,7 @@ type TestReportConverterSuite struct {
|
|||
vulnerabilityRecordDao scan.VulnerabilityRecordDao
|
||||
reportDao scan.DAO
|
||||
registrationID string
|
||||
nc *nativeToRelationalSchemaConverter
|
||||
}
|
||||
|
||||
// SetupTest prepares env for test cases.
|
||||
|
|
@ -318,6 +320,7 @@ func TestReportConverterTests(t *testing.T) {
|
|||
|
||||
// SetupSuite sets up the report converter suite test cases
|
||||
func (suite *TestReportConverterSuite) SetupSuite() {
|
||||
suite.nc = &nativeToRelationalSchemaConverter{dao: scan.NewVulnerabilityRecordDao()}
|
||||
suite.rc = NewNativeToRelationalSchemaConverter()
|
||||
suite.Suite.SetupSuite()
|
||||
suite.vulnerabilityRecordDao = scan.NewVulnerabilityRecordDao()
|
||||
|
|
@ -510,3 +513,76 @@ func (suite *TestReportConverterSuite) validateReportSummary(summary string, raw
|
|||
require.NoError(suite.T(), err)
|
||||
assert.Equal(suite.T(), string(data), summary)
|
||||
}
|
||||
|
||||
func (suite *TestReportConverterSuite) TestUpdateReport() {
|
||||
ctx := suite.Context()
|
||||
vuls := []*vuln.VulnerabilityItem{
|
||||
{
|
||||
Severity: "Critical", FixVersion: "2.9.1",
|
||||
},
|
||||
{
|
||||
Severity: "Critical",
|
||||
},
|
||||
{
|
||||
Severity: "High",
|
||||
},
|
||||
{
|
||||
Severity: "Medium",
|
||||
},
|
||||
{
|
||||
Severity: "Low",
|
||||
},
|
||||
{
|
||||
Severity: "None",
|
||||
},
|
||||
{
|
||||
Severity: "Unknown",
|
||||
},
|
||||
}
|
||||
rp := &scan.Report{
|
||||
Digest: "d1001",
|
||||
RegistrationUUID: "ruuid",
|
||||
MimeType: v1.MimeTypeGenericVulnerabilityReport,
|
||||
Report: sampleReportWithMixedSeverity,
|
||||
StartTime: time.Now(),
|
||||
EndTime: time.Now().Add(1000),
|
||||
UUID: "reportUUID3",
|
||||
}
|
||||
id, err := suite.reportDao.Create(ctx, rp)
|
||||
suite.NoError(err)
|
||||
suite.True(id > 0)
|
||||
err = suite.nc.updateReport(ctx, vuls, rp.UUID)
|
||||
suite.NoError(err)
|
||||
rpts, err := suite.reportDao.List(ctx, q.New(q.KeyWords{"UUID": rp.UUID}))
|
||||
suite.NoError(err)
|
||||
suite.Equal(1, len(rpts))
|
||||
suite.Equal(int64(2), rpts[0].CriticalCnt)
|
||||
suite.Equal(int64(1), rpts[0].HighCnt)
|
||||
suite.Equal(int64(1), rpts[0].MediumCnt)
|
||||
suite.Equal(int64(1), rpts[0].LowCnt)
|
||||
suite.Equal(int64(1), rpts[0].NoneCnt)
|
||||
suite.Equal(int64(1), rpts[0].UnknownCnt)
|
||||
suite.Equal(int64(1), rpts[0].FixableCnt)
|
||||
}
|
||||
|
||||
func Test_parseScoreFromVendorAttribute(t *testing.T) {
|
||||
type args struct {
|
||||
vendorAttribute string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
wantNvdV3Score float64
|
||||
}{
|
||||
{"normal", args{`{"CVSS":{"nvd":{"V2Score":4.3,"V2Vector":"AV:N/AC:M/Au:N/C:N/I:N/A:P","V3Score":6.5,"V3Vector":"CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:N/A:H"}}}`}, 6.5},
|
||||
{"both", args{`{"CVSS":{"nvd":{"V3Score":5.5,"V3Vector":"CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:N/I:N/A:H"},"redhat":{"V3Score":6.2,"V3Vector":"CVSS:3.1/AV:L/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"}}}`}, 5.5},
|
||||
{"both2", args{`{"CVSS":{"nvd":{"V2Score":7.2,"V2Vector":"AV:L/AC:L/Au:N/C:C/I:C/A:C","V3Score":7.8,"V3Vector":"CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H"},"redhat":{"V3Score":7.8,"V3Vector":"CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H"}}}`}, 7.8},
|
||||
{"none", args{`{"CVSS":{"nvd":{"V2Score":7.2,"V2Vector":"AV:L/AC:L/Au:N/C:C/I:C/A:C","V3Vector":"CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H"},"redhat":{"V3Vector":"CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H"}}}`}, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotNvdV3Score := parseScoreFromVendorAttribute(context.Background(), tt.args.vendorAttribute)
|
||||
assert.Equalf(t, tt.wantNvdV3Score, gotNvdV3Score, "parseScoreFromVendorAttribute(%v)", tt.args.vendorAttribute)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -101,6 +101,9 @@ type Manager interface {
|
|||
// []*scan.Report : report list
|
||||
// error : non nil error if any errors occurred
|
||||
List(ctx context.Context, query *q.Query) ([]*scan.Report, error)
|
||||
|
||||
// Update update report information
|
||||
Update(ctx context.Context, r *scan.Report, cols ...string) error
|
||||
}
|
||||
|
||||
// basicManager is a default implementation of report manager.
|
||||
|
|
@ -219,3 +222,7 @@ func (bm *basicManager) DeleteByDigests(ctx context.Context, digests ...string)
|
|||
func (bm *basicManager) List(ctx context.Context, query *q.Query) ([]*scan.Report, error) {
|
||||
return bm.dao.List(ctx, query)
|
||||
}
|
||||
|
||||
func (bm *basicManager) Update(ctx context.Context, r *scan.Report, cols ...string) error {
|
||||
return bm.dao.Update(ctx, r, cols...)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -127,6 +127,27 @@ func (_m *Manager) List(ctx context.Context, query *q.Query) ([]*scan.Report, er
|
|||
return r0, r1
|
||||
}
|
||||
|
||||
// Update provides a mock function with given fields: ctx, r, cols
|
||||
func (_m *Manager) Update(ctx context.Context, r *scan.Report, cols ...string) error {
|
||||
_va := make([]interface{}, len(cols))
|
||||
for _i := range cols {
|
||||
_va[_i] = cols[_i]
|
||||
}
|
||||
var _ca []interface{}
|
||||
_ca = append(_ca, ctx, r)
|
||||
_ca = append(_ca, _va...)
|
||||
ret := _m.Called(_ca...)
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *scan.Report, ...string) error); ok {
|
||||
r0 = rf(ctx, r, cols...)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// UpdateReportData provides a mock function with given fields: ctx, uuid, _a2
|
||||
func (_m *Manager) UpdateReportData(ctx context.Context, uuid string, _a2 string) error {
|
||||
ret := _m.Called(ctx, uuid, _a2)
|
||||
|
|
|
|||
Loading…
Reference in New Issue