fix: export cve for image manifest list (#17333)

1. Fix export cve for image manifest list
2. Remove row_id column in csv file
3. Update cve execution swagger API description

Closes: #17331,#17330,#17335,#17334

Signed-off-by: chlins <chenyuzh@vmware.com>
This commit is contained in:
Chenyu Zhang 2022-08-05 20:07:02 +08:00 committed by GitHub
parent bd102fbf7d
commit 49d73fa57d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 22 additions and 16 deletions

View File

@ -5656,7 +5656,7 @@ paths:
/export/cve/executions: /export/cve/executions:
get: get:
summary: Get a list of specific scan data export execution jobs for a specified user summary: Get a list of specific scan data export execution jobs for a specified user
description: Get the scan data export execution specified by ID description: Get a list of specific scan data export execution jobs for a specified user
tags: tags:
- scan data export - scan data export
operationId: getScanDataExportExecutionList operationId: getScanDataExportExecutionList

View File

@ -261,8 +261,6 @@ func (sde *ScanDataExport) writeCsvFile(ctx job.Context, params job.Parameters,
} }
exportParams.PageNumber = exportParams.PageNumber + 1 exportParams.PageNumber = exportParams.PageNumber + 1
exportParams.RowNumOffset = exportParams.RowNumOffset + int64(len(data))
// break earlier if this is last page // break earlier if this is last page
if len(data) < int(exportParams.PageSize) { if len(data) < int(exportParams.PageSize) {
break break

View File

@ -628,7 +628,6 @@ func (suite *ScanDataExportJobTestSuite) createDataRecords(numRecs int, ownerId
data := make([]export.Data, 0) data := make([]export.Data, 0)
for i := 1; i <= numRecs; i++ { for i := 1; i <= numRecs; i++ {
dataRec := export.Data{ dataRec := export.Data{
ID: int64(i),
ScannerName: fmt.Sprintf("TestScanner%d", i), ScannerName: fmt.Sprintf("TestScanner%d", i),
Repository: fmt.Sprintf("Repository%d", i), Repository: fmt.Sprintf("Repository%d", i),
ArtifactDigest: fmt.Sprintf("Digest%d", i), ArtifactDigest: fmt.Sprintf("Digest%d", i),

View File

@ -72,7 +72,6 @@ func (suite *ExportDataSelectorTestSuite) createDataRecords(numRecs int, ownerId
data := make([]Data, 0) data := make([]Data, 0)
for i := 1; i <= numRecs; i++ { for i := 1; i <= numRecs; i++ {
dataRec := Data{ dataRec := Data{
ID: int64(i),
ScannerName: fmt.Sprintf("TestScanner%d", i), ScannerName: fmt.Sprintf("TestScanner%d", i),
Repository: fmt.Sprintf("Repository%d", i), Repository: fmt.Sprintf("Repository%d", i),
ArtifactDigest: fmt.Sprintf("Digest%d", i), ArtifactDigest: fmt.Sprintf("Digest%d", i),

View File

@ -12,6 +12,7 @@ import (
"github.com/goharbor/harbor/src/lib/selector" "github.com/goharbor/harbor/src/lib/selector"
"github.com/goharbor/harbor/src/lib/selector/selectors/doublestar" "github.com/goharbor/harbor/src/lib/selector/selectors/doublestar"
"github.com/goharbor/harbor/src/pkg" "github.com/goharbor/harbor/src/pkg"
artpkg "github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/project" "github.com/goharbor/harbor/src/pkg/project"
"github.com/goharbor/harbor/src/pkg/project/models" "github.com/goharbor/harbor/src/pkg/project/models"
"github.com/goharbor/harbor/src/pkg/repository" "github.com/goharbor/harbor/src/pkg/repository"
@ -136,7 +137,22 @@ func (dfp *DefaultFilterProcessor) ProcessTagFilter(ctx context.Context, filter
return nil, err return nil, err
} }
arts = append(arts, repoArts...) for _, art := range repoArts {
if art.IsImageIndex() {
for _, ref := range art.References {
arts = append(arts, &artifact.Artifact{
Artifact: artpkg.Artifact{
ID: ref.ChildID,
Digest: ref.ChildDigest,
},
Tags: art.Tags,
Labels: art.Labels,
})
}
}
arts = append(arts, art)
}
} }
// return earlier if no tag filter // return earlier if no tag filter
if filter == "" { if filter == "" {

View File

@ -14,15 +14,13 @@ import (
const ( const (
// This sql template aims to select vuln data from database, // This sql template aims to select vuln data from database,
// which receive two parameters: // which receive one parameter:
// 1. rowNum offset // 1. artifacts id sets
// 2. artifacts id sets
// consider for performance, the caller will slice the artifact ids to multi // consider for performance, the caller will slice the artifact ids to multi
// groups if it's length over limit, so rowNum offset is designed to ensure the // groups if it's length over limit, so rowNum offset is designed to ensure the
// final row id is sequence in the final output csv file. // final row id is sequence in the final output csv file.
VulnScanReportQueryTemplate = ` VulnScanReportQueryTemplate = `
select select
row_number() over() + %d as result_row_id,
artifact.digest as artifact_digest, artifact.digest as artifact_digest,
artifact.repository_id, artifact.repository_id,
artifact.repository_name, artifact.repository_name,
@ -66,9 +64,6 @@ var (
// Params specifies the filters for controlling the scan data export process // Params specifies the filters for controlling the scan data export process
type Params struct { type Params struct {
// rowNumber offset
RowNumOffset int64
// cve ids // cve ids
CVEIds string CVEIds string
@ -139,7 +134,7 @@ func (em *exportManager) buildQuery(ctx context.Context, params Params) (beego_o
} }
} }
sql := fmt.Sprintf(VulnScanReportQueryTemplate, params.RowNumOffset, artIDs) sql := fmt.Sprintf(VulnScanReportQueryTemplate, artIDs)
ormer, err := orm.FromContext(ctx) ormer, err := orm.FromContext(ctx)
if err != nil { if err != nil {
return nil, err return nil, err

View File

@ -9,8 +9,6 @@ import (
// Data models a single row of the exported scan vulnerability data // Data models a single row of the exported scan vulnerability data
type Data struct { type Data struct {
ID int64 `orm:"column(result_row_id)" csv:"RowId"`
ScannerName string `orm:"column(scanner_name)" csv:"Scanner"`
Repository string `orm:"column(repository_name)" csv:"Repository"` Repository string `orm:"column(repository_name)" csv:"Repository"`
ArtifactDigest string `orm:"column(artifact_digest)" csv:"Artifact Digest"` ArtifactDigest string `orm:"column(artifact_digest)" csv:"Artifact Digest"`
CVEId string `orm:"column(cve_id)" csv:"CVE"` CVEId string `orm:"column(cve_id)" csv:"CVE"`
@ -20,6 +18,7 @@ type Data struct {
Severity string `orm:"column(severity)" csv:"Severity"` Severity string `orm:"column(severity)" csv:"Severity"`
CWEIds string `orm:"column(cwe_ids)" csv:"CWE Ids"` CWEIds string `orm:"column(cwe_ids)" csv:"CWE Ids"`
AdditionalData string `orm:"column(vendor_attributes)" csv:"Additional Data"` AdditionalData string `orm:"column(vendor_attributes)" csv:"Additional Data"`
ScannerName string `orm:"column(scanner_name)" csv:"Scanner"`
} }
// Request encapsulates the filters to be provided when exporting the data for a scan. // Request encapsulates the filters to be provided when exporting the data for a scan.