Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-26 04:05:40 +01:00
Code Issues Projects Releases Wiki Activity

fix: skip to push system artifact for empty CSV file (#17816)

Browse Source 
1. Skip to push system artifact to the distribution when the exported CSV file is empty.
2. Add status message for cve export execution.

Signed-off-by: chlins <chenyuzh@vmware.com>
This commit is contained in:
Chlins Zhang 2022-11-18 14:16:36 +08:00 committed by GitHub
parent 62223bd36d
commit 321a9abfb3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 94 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/controller/scandataexport/execution.go
View File

@ -185,9 +185,7 @@ func (c *controller) convertToExportExecStatus(ctx context.Context, exec *task.E
if userName, ok := exec.ExtraAttrs[export.UserNameAttribute]; ok { if userName, ok := exec.ExtraAttrs[export.UserNameAttribute]; ok {
execStatus.UserName = userName.(string) execStatus.UserName = userName.(string)
} }
// Do not define the 'status_message' as const because this is not the final solution, if statusMessage, ok := exec.ExtraAttrs[export.StatusMessageAttribute]; ok {
// should refactor this part.
if statusMessage, ok := exec.ExtraAttrs["status_message"]; ok {
execStatus.StatusMessage = statusMessage.(string) execStatus.StatusMessage = statusMessage.(string)
} }
artifactExists := c.isCsvArtifactPresent(ctx, exec.ID, execStatus.ExportDataDigest) artifactExists := c.isCsvArtifactPresent(ctx, exec.ID, execStatus.ExportDataDigest)

27
src/jobservice/job/impl/scandataexport/scan_data_export.go
View File

@ -110,24 +110,25 @@ func (sde *ScanDataExport) Run(ctx job.Context, params job.Parameters) error {
return err return err
} }
logger.Infof("Export Job Id = %v. CSV file size: %d", params["JobId"], stat.Size()) logger.Infof("Export Job Id = %v. CSV file size: %d", params["JobId"], stat.Size())
// earlier return and update status message if the file size is 0, unnecessary to push a empty system artifact.
if stat.Size() == 0 {
extra := map[string]interface{}{
export.StatusMessageAttribute: "No vulnerabilities found or matched",
}
updateErr := sde.updateExecAttributes(ctx, params, extra)
if updateErr != nil {
logger.Errorf("Export Job Id = %v. Error when updating the exec extra attributes 'status_message' to 'No vulnerabilities found or matched': %v", params["JobId"], updateErr)
}
logger.Infof("Export Job Id = %v. Exported CSV file is empty, skip to push system artifact, exit job", params["JobId"])
return nil
}
csvExportArtifactRecord := model.SystemArtifact{Repository: repositoryName, Digest: hash.String(), Size: stat.Size(), Type: "ScanData_CSV", Vendor: strings.ToLower(export.Vendor)} csvExportArtifactRecord := model.SystemArtifact{Repository: repositoryName, Digest: hash.String(), Size: stat.Size(), Type: "ScanData_CSV", Vendor: strings.ToLower(export.Vendor)}
artID, err := sde.sysArtifactMgr.Create(ctx.SystemContext(), &csvExportArtifactRecord, csvFile) artID, err := sde.sysArtifactMgr.Create(ctx.SystemContext(), &csvExportArtifactRecord, csvFile)
if err != nil { if err != nil {
logger.Errorf( logger.Errorf(
"Export Job Id = %v. Error when persisting report file %s to persistent storage: %v", params["JobId"], fileName, err) "Export Job Id = %v. Error when persisting report file %s to persistent storage: %v", params["JobId"], fileName, err)
// NOTICE: this is a tentative solution to resolve error to push empty blob to S3 storage driver,
// should unify the behaviour for different drivers.
// Temporary set the status message to extra attributes, then the API handler will fetch it and combined to response for better experience.
if stat.Size() == 0 {
extra := map[string]interface{}{
"status_message": "No vulnerabilities found or matched",
}
updateErr := sde.updateExecAttributes(ctx, params, extra)
if updateErr != nil {
logger.Errorf("Export Job Id = %v. Error when updating the exec extra attributes 'status_message' to 'No vulnerabilities found or matched': %v", params["JobId"], updateErr)
}
}
return err return err
} }

90
src/jobservice/job/impl/scandataexport/scan_data_export_test.go
View File

@ -70,10 +70,13 @@ func (suite *ScanDataExportJobTestSuite) SetupTest() {
func (suite *ScanDataExportJobTestSuite) TestRun() { func (suite *ScanDataExportJobTestSuite) TestRun() {
data := suite.createDataRecords(3, 1) data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(MockDigest), nil) mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(MockDigest), nil)
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{1}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{1}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return([]*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1}}}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessLabelFilter").Return([]*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1}}}, nil).Once()
execAttrs := make(map[string]interface{}) execAttrs := make(map[string]interface{})
execAttrs[export.JobNameAttribute] = "test-job" execAttrs[export.JobNameAttribute] = "test-job"
@ -83,6 +86,7 @@ func (suite *ScanDataExportJobTestSuite) TestRun() {
params := job.Parameters{} params := job.Parameters{}
params[export.JobModeKey] = export.JobModeExport params[export.JobModeKey] = export.JobModeExport
params["JobId"] = JobId params["JobId"] = JobId
params["Request"] = map[string]interface{}{}
ctx := &mockjobservice.MockJobContext{} ctx := &mockjobservice.MockJobContext{}
err := suite.job.Run(ctx, params) err := suite.job.Run(ctx, params)
@ -106,19 +110,9 @@ func (suite *ScanDataExportJobTestSuite) TestRun() {
} }
func (suite *ScanDataExportJobTestSuite) TestRunCreateSysArtError() { func (suite *ScanDataExportJobTestSuite) TestRunWithEmptyData() {
oldSysArtMgr := suite.sysArtifactMgr
defer func() {
suite.job.sysArtifactMgr = oldSysArtMgr
}()
sysArtMgr := &systemartifacttesting.Manager{}
suite.job.sysArtifactMgr = sysArtMgr
sysArtMgr.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(int64(-1), errors.New("create sys artifact error")).Once()
// mock create system artifact error when file is empty
var data []export.Data var data []export.Data
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(MockDigest), nil) mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(MockDigest), nil)
execAttrs := make(map[string]interface{}) execAttrs := make(map[string]interface{})
@ -132,7 +126,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunCreateSysArtError() {
ctx := &mockjobservice.MockJobContext{} ctx := &mockjobservice.MockJobContext{}
err := suite.job.Run(ctx, params) err := suite.job.Run(ctx, params)
suite.Error(err) suite.NoError(err)
extraAttrsMatcher := testifymock.MatchedBy(func(attrsMap map[string]interface{}) bool { extraAttrsMatcher := testifymock.MatchedBy(func(attrsMap map[string]interface{}) bool {
return attrsMap["status_message"] == "No vulnerabilities found or matched" && attrsMap[export.JobNameAttribute] == "test-job" && attrsMap[export.UserNameAttribute] == "test-user" return attrsMap["status_message"] == "No vulnerabilities found or matched" && attrsMap[export.JobNameAttribute] == "test-job" && attrsMap[export.UserNameAttribute] == "test-user"
@ -142,9 +136,12 @@ func (suite *ScanDataExportJobTestSuite) TestRunCreateSysArtError() {
func (suite *ScanDataExportJobTestSuite) TestRunAttributeUpdateError() { func (suite *ScanDataExportJobTestSuite) TestRunAttributeUpdateError() {
data := suite.createDataRecords(3, 1) data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once() mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{1}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{1}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return([]*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1}}}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessLabelFilter").Return([]*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1}}}, nil).Once()
mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(MockDigest), nil) mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(MockDigest), nil)
execAttrs := make(map[string]interface{}) execAttrs := make(map[string]interface{})
@ -155,6 +152,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunAttributeUpdateError() {
params := job.Parameters{} params := job.Parameters{}
params[export.JobModeKey] = export.JobModeExport params[export.JobModeKey] = export.JobModeExport
params["JobId"] = JobId params["JobId"] = JobId
params["Request"] = map[string]interface{}{}
ctx := &mockjobservice.MockJobContext{} ctx := &mockjobservice.MockJobContext{}
err := suite.job.Run(ctx, params) err := suite.job.Run(ctx, params)
@ -199,7 +197,7 @@ func (suite *ScanDataExportJobTestSuite) TestExtractCriteria() {
func (suite *ScanDataExportJobTestSuite) TestRunWithCriteria() { func (suite *ScanDataExportJobTestSuite) TestRunWithCriteria() {
{ {
data := suite.createDataRecords(3, 1) data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
@ -262,7 +260,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteria() {
{ {
mock.OnAnything(suite.sysArtifactMgr, "Create").Return(int64(1), nil).Once() mock.OnAnything(suite.sysArtifactMgr, "Create").Return(int64(1), nil).Once()
data := suite.createDataRecords(3, 1) data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(MockDigest), nil) mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(MockDigest), nil)
@ -325,7 +323,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteria() {
func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForProjectIdFilter() { func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForProjectIdFilter() {
{ {
data := suite.createDataRecords(3, 1) data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
@ -338,11 +336,9 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForProjectIdFilter()
execAttrs[export.UserNameAttribute] = "test-user" execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once() mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once()
repoCandidate1 := &selector.Candidate{NamespaceID: 1}
repoCandidates := []*selector.Candidate{repoCandidate1}
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return(nil, errors.New("test error")).Once() mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return(nil, errors.New("test error")).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return(repoCandidates, nil) mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return(nil, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(repoCandidates, nil) mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(nil, nil).Once()
criteria := export.Request{ criteria := export.Request{
CVEIds: "CVE-123", CVEIds: "CVE-123",
@ -378,7 +374,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForProjectIdFilter()
// empty list of projects // empty list of projects
{ {
data := suite.createDataRecords(3, 1) data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
@ -391,11 +387,9 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForProjectIdFilter()
execAttrs[export.UserNameAttribute] = "test-user" execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once() mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once()
repoCandidate1 := &selector.Candidate{NamespaceID: 1}
repoCandidates := []*selector.Candidate{repoCandidate1}
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{}, nil).Once() mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return(repoCandidates, nil) mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{1}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(repoCandidates, nil) mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return([]*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1}}}, nil).Once()
criteria := export.Request{ criteria := export.Request{
CVEIds: "CVE-123", CVEIds: "CVE-123",
@ -420,7 +414,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForProjectIdFilter()
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool { sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
}) })
suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything) suite.sysArtifactMgr.AssertNotCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), mock.Anything) suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), mock.Anything)
_, err = os.Stat("/tmp/scandata_export_100.csv") _, err = os.Stat("/tmp/scandata_export_100.csv")
@ -434,7 +428,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForProjectIdFilter()
func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilter() { func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilter() {
{ {
data := suite.createDataRecords(3, 1) data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
@ -447,11 +441,9 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilte
execAttrs[export.UserNameAttribute] = "test-user" execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once() mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once()
repoCandidate1 := &selector.Candidate{NamespaceID: 1}
repoCandidates := []*selector.Candidate{repoCandidate1}
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{1}, errors.New("test error")).Once() mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{1}, errors.New("test error")).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return(nil, errors.New("test error")) mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{1}, errors.New("test error")).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(repoCandidates, nil) mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return([]*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1}}}, nil).Once()
criteria := export.Request{ criteria := export.Request{
CVEIds: "CVE-123", CVEIds: "CVE-123",
@ -487,7 +479,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilte
// empty list of repo ids // empty list of repo ids
{ {
data := suite.createDataRecords(3, 1) data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
@ -500,10 +492,9 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilte
execAttrs[export.UserNameAttribute] = "test-user" execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once() mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once()
repoCandidates := make([]*selector.Candidate, 0)
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{}, nil).Once() mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return(repoCandidates, nil) mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(repoCandidates, nil) mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return([]*artifact.Artifact{}, nil).Once()
criteria := export.Request{ criteria := export.Request{
CVEIds: "CVE-123", CVEIds: "CVE-123",
@ -528,7 +519,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilte
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool { sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
}) })
suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything) suite.sysArtifactMgr.AssertNotCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), mock.Anything) suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), mock.Anything)
_, err = os.Stat("/tmp/scandata_export_100.csv") _, err = os.Stat("/tmp/scandata_export_100.csv")
@ -541,7 +532,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilte
func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithTagFilter() { func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithTagFilter() {
{ {
data := suite.createDataRecords(3, 1) data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
@ -554,11 +545,9 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithT
execAttrs[export.UserNameAttribute] = "test-user" execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once() mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once()
repoCandidate1 := &selector.Candidate{NamespaceID: 1}
repoCandidates := []*selector.Candidate{repoCandidate1}
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{1}, errors.New("test error")).Once() mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{1}, errors.New("test error")).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return(repoCandidates, nil) mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{1}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(nil, errors.New("test error")) mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(nil, errors.New("test error")).Once()
criteria := export.Request{ criteria := export.Request{
CVEIds: "CVE-123", CVEIds: "CVE-123",
@ -594,7 +583,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithT
// empty list of repo ids after applying tag filters // empty list of repo ids after applying tag filters
{ {
data := suite.createDataRecords(3, 1) data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
@ -607,10 +596,9 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithT
execAttrs[export.UserNameAttribute] = "test-user" execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once() mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once()
repoCandidates := make([]*selector.Candidate, 0)
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{}, nil).Once() mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return(repoCandidates, nil) mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(make([]*selector.Candidate, 0), nil) mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(nil, nil).Once()
criteria := export.Request{ criteria := export.Request{
CVEIds: "CVE-123", CVEIds: "CVE-123",
@ -635,7 +623,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithT
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool { sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
}) })
suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything) suite.sysArtifactMgr.AssertNotCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), mock.Anything) suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), mock.Anything)
_, err = os.Stat("/tmp/scandata_export_100.csv") _, err = os.Stat("/tmp/scandata_export_100.csv")
@ -647,7 +635,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithT
} }
func (suite *ScanDataExportJobTestSuite) TestExportDigestCalculationErrorsOut() { func (suite *ScanDataExportJobTestSuite) TestExportDigestCalculationErrorsOut() {
data := suite.createDataRecords(3, 1) data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once() mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(""), errors.New("test error")) mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(""), errors.New("test error"))
@ -677,7 +665,7 @@ func (suite *ScanDataExportJobTestSuite) TearDownTest() {
suite.NoError(err) suite.NoError(err)
} }
func (suite *ScanDataExportJobTestSuite) createDataRecords(numRecs int, ownerId int64) []export.Data { func (suite *ScanDataExportJobTestSuite) createDataRecords(numRecs int) []export.Data {
data := make([]export.Data, 0) data := make([]export.Data, 0)
for i := 1; i <= numRecs; i++ { for i := 1; i <= numRecs; i++ {
dataRec := export.Data{ dataRec := export.Data{

21
src/pkg/scan/export/constants.go
View File

@ -4,14 +4,15 @@ package export
type CsvJobVendorID string type CsvJobVendorID string
const ( const (
ProjectIDsAttribute = "project_ids" ProjectIDsAttribute = "project_ids"
JobNameAttribute = "job_name" JobNameAttribute = "job_name"
UserNameAttribute = "user_name" UserNameAttribute = "user_name"
ScanDataExportDir = "/var/scandata_exports" StatusMessageAttribute = "status_message"
QueryPageSize = 100000 ScanDataExportDir = "/var/scandata_exports"
ArtifactGroupSize = 10000 QueryPageSize = 100000
DigestKey = "artifact_digest" ArtifactGroupSize = 10000
CreateTimestampKey = "create_ts" DigestKey = "artifact_digest"
Vendor = "SCAN_DATA_EXPORT" CreateTimestampKey = "create_ts"
CsvJobVendorIDKey = CsvJobVendorID("vendorId") Vendor = "SCAN_DATA_EXPORT"
CsvJobVendorIDKey = CsvJobVendorID("vendorId")
) )

14
src/server/v2.0/handler/scanexport.go
View File

@ -160,13 +160,6 @@ func (se *scanDataExportAPI) DownloadScanData(ctx context.Context, params operat
return se.SendError(ctx, err) return se.SendError(ctx, err)
} }
// check if the CSV artifact for the execution exists
if !execution.FilePresent {
return middleware.ResponderFunc(func(writer http.ResponseWriter, producer runtime.Producer) {
writer.WriteHeader(http.StatusNotFound)
})
}
// check if the execution being downloaded is owned by the current user // check if the execution being downloaded is owned by the current user
secContext, err := se.GetSecurityContext(ctx) secContext, err := se.GetSecurityContext(ctx)
if err != nil { if err != nil {
@ -179,6 +172,13 @@ func (se *scanDataExportAPI) DownloadScanData(ctx context.Context, params operat
}) })
} }
// check if the CSV artifact for the execution exists
if !execution.FilePresent {
return middleware.ResponderFunc(func(writer http.ResponseWriter, producer runtime.Producer) {
writer.WriteHeader(http.StatusNotFound)
})
}
repositoryName := fmt.Sprintf("scandata_export_%v", params.ExecutionID) repositoryName := fmt.Sprintf("scandata_export_%v", params.ExecutionID)
file, err := se.sysArtifactMgr.Read(ctx, strings.ToLower(export.Vendor), repositoryName, execution.ExportDataDigest) file, err := se.sysArtifactMgr.Read(ctx, strings.ToLower(export.Vendor), repositoryName, execution.ExportDataDigest)
if err != nil { if err != nil {

2
src/server/v2.0/handler/scanexport_test.go
View File

@ -514,7 +514,7 @@ func (suite *ScanExportTestSuite) TestDownloadScanDataNoCsvFilePresent() {
StartTime: startTime, StartTime: startTime,
EndTime: endTime, EndTime: endTime,
ExportDataDigest: "datadigest", ExportDataDigest: "datadigest",
UserName: "test-user", UserName: "test-user1",
FilePresent: false, FilePresent: false,
} }
mock.OnAnything(suite.scanExportCtl, "GetExecution").Return(execution, nil) mock.OnAnything(suite.scanExportCtl, "GetExecution").Return(execution, nil)

35
tests/apitests/python/test_scan_data_export.py
View File

@ -14,6 +14,7 @@ from library.artifact import Artifact
from library.scan import Scan from library.scan import Scan
from library.repository import push_self_build_image_to_project from library.repository import push_self_build_image_to_project
class TestScanDataExport(unittest.TestCase): class TestScanDataExport(unittest.TestCase):
@suppress_urllib3_warning @suppress_urllib3_warning
@ -45,25 +46,26 @@ class TestScanDataExport(unittest.TestCase):
11. Verify that the export scan data execution triggered by the user (UA) cannot be download by other users; 11. Verify that the export scan data execution triggered by the user (UA) cannot be download by other users;
12. User (UA) should be able to download the triggered export scan data execution 12. User (UA) should be able to download the triggered export scan data execution
13. Verify that the downloaded export scan data execution cannot be downloaded again 13. Verify that the downloaded export scan data execution cannot be downloaded again
14. Verify the status message if no cve found or matched
""" """
url = ADMIN_CLIENT["endpoint"] url = ADMIN_CLIENT["endpoint"]
user_password = "Aa123456" user_password = "Aa123456"
# 1. Create user(UA) # 1. Create user(UA)
user_id, user_name = self.user.create_user(user_password = user_password, **ADMIN_CLIENT) user_id, user_name = self.user.create_user(user_password=user_password, **ADMIN_CLIENT)
user_client = dict(endpoint = url, username = user_name, password = user_password) user_client = dict(endpoint=url, username=user_name, password=user_password)
# 2.1. Create private project(PA) by user(UA) # 2.1. Create private project(PA) by user(UA)
project_id, project_name = self.project.create_project(metadata = {"public": "false"}, **user_client) project_id, project_name = self.project.create_project(metadata={"public": "false"}, **user_client)
# 2.2. Get private project of uesr-001, uesr-001 can see only one private project which is project-001 # 2.2. Get private project of uesr-001, uesr-001 can see only one private project which is project-001
self.project.projects_should_exist(dict(public=False), expected_count = 1, expected_project_id = project_id, **user_client) self.project.projects_should_exist(dict(public=False), expected_count=1, expected_project_id=project_id, **user_client)
# 3. Push a new image(IA) in project(PA) by user(UA) # 3. Push a new image(IA) in project(PA) by user(UA)
push_self_build_image_to_project(project_name, harbor_server, user_name, user_password, self.image, self.tag) push_self_build_image_to_project(project_name, harbor_server, user_name, user_password, self.image, self.tag)
# 4. Send scan image command and get tag(TA) information to check scan result, it should be finished # 4. Send scan image command and get tag(TA) information to check scan result, it should be finished
self.scan.scan_artifact(project_name, self.image, self.tag, **user_client) self.scan.scan_artifact(project_name, self.image, self.tag, **user_client)
self.artifact.check_image_scan_result(project_name, self.image, self.tag, with_scan_overview = True, **user_client) self.artifact.check_image_scan_result(project_name, self.image, self.tag, with_scan_overview=True, **user_client)
# 5. Verify trigger export scan data execution but does not specify Scan-Data-Type status code should be 422 # 5. Verify trigger export scan data execution but does not specify Scan-Data-Type status code should be 422
self.scan_data_export.export_scan_data("", projects=[project_id], expect_status_code=422, expect_response_body="X-Scan-Data-Type in header is required") self.scan_data_export.export_scan_data("", projects=[project_id], expect_status_code=422, expect_response_body="X-Scan-Data-Type in header is required")
@ -90,26 +92,31 @@ class TestScanDataExport(unittest.TestCase):
self.assertEqual(user_name, execution_list.items[0].user_name) self.assertEqual(user_name, execution_list.items[0].user_name)
# 10. Wait for the export scan data execution to succeed # 10. Wait for the export scan data execution to succeed
executio_status = None execution = None
for i in range(5): for i in range(5):
print("wait for the job to finish:", i) print("wait for the job to finish:", i)
execution = self.scan_data_export.get_scan_data_export_execution(execution_id, **user_client) execution = self.scan_data_export.get_scan_data_export_execution(execution_id, **user_client)
executio_status = execution.status if execution.status == "Success":
if executio_status == "Success":
self.assertEqual(user_name, execution.user_name) self.assertEqual(user_name, execution.user_name)
self.assertEqual(user_id, execution.user_id) self.assertEqual(user_id, execution.user_id)
break break
time.sleep(2) time.sleep(2)
self.assertEqual(executio_status, "Success") self.assertEqual(execution.status, "Success")
# 11. Verify that the export scan data execution triggered by the user (UA) cannot be download by other users # 11. Verify that the export scan data execution triggered by the user (UA) cannot be download by other users
self.scan_data_export.download_scan_data(execution_id, expect_status_code=403) self.scan_data_export.download_scan_data(execution_id, expect_status_code=403)
# 12. User (UA) should be able to download the triggered export scan data execution # The csv file will not be able to downloaded if it is empty, so only check download if the file is present, otherwise check the status message
self.scan_data_export.download_scan_data(execution_id, **user_client) if execution.file_present:
# 12. User (UA) should be able to download the triggered export scan data execution
self.scan_data_export.download_scan_data(execution_id, **user_client)
# 13. Verify that the downloaded export scan data execution cannot be downloaded again
self.scan_data_export.download_scan_data(execution_id, expect_status_code=404, **user_client)
else:
# 14. Verify the status message if no cve found or matched
self.assertEqual("No vulnerabilities found or matched", execution.status_text)
# 13. Verify that the downloaded export scan data execution cannot be downloaded again
self.scan_data_export.download_scan_data(execution_id, expect_status_code=404, **user_client)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()