From 49d73fa57dd74766ae7cd1083b53d50ed6624489 Mon Sep 17 00:00:00 2001 From: Chenyu Zhang Date: Fri, 5 Aug 2022 20:07:02 +0800 Subject: [PATCH] fix: export cve for image manifest list (#17333) 1. Fix export cve for image manifest list 2. Remove row_id column in csv file 3. Update cve execution swagger API description Closes: #17331,#17330,#17335,#17334 Signed-off-by: chlins --- api/v2.0/swagger.yaml | 2 +- .../impl/scandataexport/scan_data_export.go | 2 -- .../scandataexport/scan_data_export_test.go | 1 - .../scan/export/export_data_selector_test.go | 1 - src/pkg/scan/export/filter_processor.go | 18 +++++++++++++++++- src/pkg/scan/export/manager.go | 11 +++-------- src/pkg/scan/export/model.go | 3 +-- 7 files changed, 22 insertions(+), 16 deletions(-) diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 6fca7b512..313325114 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -5656,7 +5656,7 @@ paths: /export/cve/executions: get: summary: Get a list of specific scan data export execution jobs for a specified user - description: Get the scan data export execution specified by ID + description: Get a list of specific scan data export execution jobs for a specified user tags: - scan data export operationId: getScanDataExportExecutionList diff --git a/src/jobservice/job/impl/scandataexport/scan_data_export.go b/src/jobservice/job/impl/scandataexport/scan_data_export.go index 3d7791831..6eaf69c30 100644 --- a/src/jobservice/job/impl/scandataexport/scan_data_export.go +++ b/src/jobservice/job/impl/scandataexport/scan_data_export.go @@ -261,8 +261,6 @@ func (sde *ScanDataExport) writeCsvFile(ctx job.Context, params job.Parameters, } exportParams.PageNumber = exportParams.PageNumber + 1 - exportParams.RowNumOffset = exportParams.RowNumOffset + int64(len(data)) - // break earlier if this is last page if len(data) < int(exportParams.PageSize) { break diff --git a/src/jobservice/job/impl/scandataexport/scan_data_export_test.go b/src/jobservice/job/impl/scandataexport/scan_data_export_test.go index 9d421a8ad..2cabe6dfa 100644 --- a/src/jobservice/job/impl/scandataexport/scan_data_export_test.go +++ b/src/jobservice/job/impl/scandataexport/scan_data_export_test.go @@ -628,7 +628,6 @@ func (suite *ScanDataExportJobTestSuite) createDataRecords(numRecs int, ownerId data := make([]export.Data, 0) for i := 1; i <= numRecs; i++ { dataRec := export.Data{ - ID: int64(i), ScannerName: fmt.Sprintf("TestScanner%d", i), Repository: fmt.Sprintf("Repository%d", i), ArtifactDigest: fmt.Sprintf("Digest%d", i), diff --git a/src/pkg/scan/export/export_data_selector_test.go b/src/pkg/scan/export/export_data_selector_test.go index bb453401b..1646b48ca 100644 --- a/src/pkg/scan/export/export_data_selector_test.go +++ b/src/pkg/scan/export/export_data_selector_test.go @@ -72,7 +72,6 @@ func (suite *ExportDataSelectorTestSuite) createDataRecords(numRecs int, ownerId data := make([]Data, 0) for i := 1; i <= numRecs; i++ { dataRec := Data{ - ID: int64(i), ScannerName: fmt.Sprintf("TestScanner%d", i), Repository: fmt.Sprintf("Repository%d", i), ArtifactDigest: fmt.Sprintf("Digest%d", i), diff --git a/src/pkg/scan/export/filter_processor.go b/src/pkg/scan/export/filter_processor.go index 5615d5afb..0e60e6adf 100644 --- a/src/pkg/scan/export/filter_processor.go +++ b/src/pkg/scan/export/filter_processor.go @@ -12,6 +12,7 @@ import ( "github.com/goharbor/harbor/src/lib/selector" "github.com/goharbor/harbor/src/lib/selector/selectors/doublestar" "github.com/goharbor/harbor/src/pkg" + artpkg "github.com/goharbor/harbor/src/pkg/artifact" "github.com/goharbor/harbor/src/pkg/project" "github.com/goharbor/harbor/src/pkg/project/models" "github.com/goharbor/harbor/src/pkg/repository" @@ -136,7 +137,22 @@ func (dfp *DefaultFilterProcessor) ProcessTagFilter(ctx context.Context, filter return nil, err } - arts = append(arts, repoArts...) + for _, art := range repoArts { + if art.IsImageIndex() { + for _, ref := range art.References { + arts = append(arts, &artifact.Artifact{ + Artifact: artpkg.Artifact{ + ID: ref.ChildID, + Digest: ref.ChildDigest, + }, + Tags: art.Tags, + Labels: art.Labels, + }) + } + } + + arts = append(arts, art) + } } // return earlier if no tag filter if filter == "" { diff --git a/src/pkg/scan/export/manager.go b/src/pkg/scan/export/manager.go index b6997ed30..5b74abc13 100644 --- a/src/pkg/scan/export/manager.go +++ b/src/pkg/scan/export/manager.go @@ -14,15 +14,13 @@ import ( const ( // This sql template aims to select vuln data from database, - // which receive two parameters: - // 1. rowNum offset - // 2. artifacts id sets + // which receive one parameter: + // 1. artifacts id sets // consider for performance, the caller will slice the artifact ids to multi // groups if it's length over limit, so rowNum offset is designed to ensure the // final row id is sequence in the final output csv file. VulnScanReportQueryTemplate = ` select - row_number() over() + %d as result_row_id, artifact.digest as artifact_digest, artifact.repository_id, artifact.repository_name, @@ -66,9 +64,6 @@ var ( // Params specifies the filters for controlling the scan data export process type Params struct { - // rowNumber offset - RowNumOffset int64 - // cve ids CVEIds string @@ -139,7 +134,7 @@ func (em *exportManager) buildQuery(ctx context.Context, params Params) (beego_o } } - sql := fmt.Sprintf(VulnScanReportQueryTemplate, params.RowNumOffset, artIDs) + sql := fmt.Sprintf(VulnScanReportQueryTemplate, artIDs) ormer, err := orm.FromContext(ctx) if err != nil { return nil, err diff --git a/src/pkg/scan/export/model.go b/src/pkg/scan/export/model.go index 5cf56896b..162cf8ac5 100644 --- a/src/pkg/scan/export/model.go +++ b/src/pkg/scan/export/model.go @@ -9,8 +9,6 @@ import ( // Data models a single row of the exported scan vulnerability data type Data struct { - ID int64 `orm:"column(result_row_id)" csv:"RowId"` - ScannerName string `orm:"column(scanner_name)" csv:"Scanner"` Repository string `orm:"column(repository_name)" csv:"Repository"` ArtifactDigest string `orm:"column(artifact_digest)" csv:"Artifact Digest"` CVEId string `orm:"column(cve_id)" csv:"CVE"` @@ -20,6 +18,7 @@ type Data struct { Severity string `orm:"column(severity)" csv:"Severity"` CWEIds string `orm:"column(cwe_ids)" csv:"CWE Ids"` AdditionalData string `orm:"column(vendor_attributes)" csv:"Additional Data"` + ScannerName string `orm:"column(scanner_name)" csv:"Scanner"` } // Request encapsulates the filters to be provided when exporting the data for a scan.