Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix: skip to push system artifact for empty CSV file (#17816) 

1. Skip to push system artifact to the distribution when the exported CSV file is empty.
2. Add status message for cve export execution.

Signed-off-by: chlins <chenyuzh@vmware.com>
This commit is contained in:
Chlins Zhang 2022-11-18 14:16:36 +08:00 committed by GitHub
parent 62223bd36d
commit 321a9abfb3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 94 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/controller/scandataexport/execution.go
View File

@ -185,9 +185,7 @@ func (c *controller) convertToExportExecStatus(ctx context.Context, exec *task.E
if userName, ok := exec.ExtraAttrs[export.UserNameAttribute]; ok {
execStatus.UserName = userName.(string)
}
// Do not define the 'status_message' as const because this is not the final solution,
// should refactor this part.
if statusMessage, ok := exec.ExtraAttrs["status_message"]; ok {
if statusMessage, ok := exec.ExtraAttrs[export.StatusMessageAttribute]; ok {
execStatus.StatusMessage = statusMessage.(string)
}
artifactExists := c.isCsvArtifactPresent(ctx, exec.ID, execStatus.ExportDataDigest)

27
src/jobservice/job/impl/scandataexport/scan_data_export.go
View File

@ -110,24 +110,25 @@ func (sde *ScanDataExport) Run(ctx job.Context, params job.Parameters) error {
return err
}
logger.Infof("Export Job Id = %v. CSV file size: %d", params["JobId"], stat.Size())
// earlier return and update status message if the file size is 0, unnecessary to push a empty system artifact.
if stat.Size() == 0 {
extra := map[string]interface{}{
export.StatusMessageAttribute: "No vulnerabilities found or matched",
}
updateErr := sde.updateExecAttributes(ctx, params, extra)
if updateErr != nil {
logger.Errorf("Export Job Id = %v. Error when updating the exec extra attributes 'status_message' to 'No vulnerabilities found or matched': %v", params["JobId"], updateErr)
}
logger.Infof("Export Job Id = %v. Exported CSV file is empty, skip to push system artifact, exit job", params["JobId"])
return nil
}
csvExportArtifactRecord := model.SystemArtifact{Repository: repositoryName, Digest: hash.String(), Size: stat.Size(), Type: "ScanData_CSV", Vendor: strings.ToLower(export.Vendor)}
artID, err := sde.sysArtifactMgr.Create(ctx.SystemContext(), &csvExportArtifactRecord, csvFile)
if err != nil {
logger.Errorf(
"Export Job Id = %v. Error when persisting report file %s to persistent storage: %v", params["JobId"], fileName, err)
// NOTICE: this is a tentative solution to resolve error to push empty blob to S3 storage driver,
// should unify the behaviour for different drivers.
// Temporary set the status message to extra attributes, then the API handler will fetch it and combined to response for better experience.
if stat.Size() == 0 {
extra := map[string]interface{}{
"status_message": "No vulnerabilities found or matched",
}
updateErr := sde.updateExecAttributes(ctx, params, extra)
if updateErr != nil {
logger.Errorf("Export Job Id = %v. Error when updating the exec extra attributes 'status_message' to 'No vulnerabilities found or matched': %v", params["JobId"], updateErr)
}
}
return err
}

90
src/jobservice/job/impl/scandataexport/scan_data_export_test.go
View File

@ -70,10 +70,13 @@ func (suite *ScanDataExportJobTestSuite) SetupTest() {
func (suite *ScanDataExportJobTestSuite) TestRun() {
data := suite.createDataRecords(3, 1)
data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(MockDigest), nil)
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{1}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{1}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return([]*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1}}}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessLabelFilter").Return([]*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1}}}, nil).Once()
execAttrs := make(map[string]interface{})
execAttrs[export.JobNameAttribute] = "test-job"
@ -83,6 +86,7 @@ func (suite *ScanDataExportJobTestSuite) TestRun() {
params := job.Parameters{}
params[export.JobModeKey] = export.JobModeExport
params["JobId"] = JobId
params["Request"] = map[string]interface{}{}
ctx := &mockjobservice.MockJobContext{}
err := suite.job.Run(ctx, params)
@ -106,19 +110,9 @@ func (suite *ScanDataExportJobTestSuite) TestRun() {
}
func (suite *ScanDataExportJobTestSuite) TestRunCreateSysArtError() {
oldSysArtMgr := suite.sysArtifactMgr
defer func() {
suite.job.sysArtifactMgr = oldSysArtMgr
}()
sysArtMgr := &systemartifacttesting.Manager{}
suite.job.sysArtifactMgr = sysArtMgr
sysArtMgr.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(int64(-1), errors.New("create sys artifact error")).Once()
// mock create system artifact error when file is empty
func (suite *ScanDataExportJobTestSuite) TestRunWithEmptyData() {
var data []export.Data
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(MockDigest), nil)
execAttrs := make(map[string]interface{})
@ -132,7 +126,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunCreateSysArtError() {
ctx := &mockjobservice.MockJobContext{}
err := suite.job.Run(ctx, params)
suite.Error(err)
suite.NoError(err)
extraAttrsMatcher := testifymock.MatchedBy(func(attrsMap map[string]interface{}) bool {
return attrsMap["status_message"] == "No vulnerabilities found or matched" && attrsMap[export.JobNameAttribute] == "test-job" && attrsMap[export.UserNameAttribute] == "test-user"
@ -142,9 +136,12 @@ func (suite *ScanDataExportJobTestSuite) TestRunCreateSysArtError() {
func (suite *ScanDataExportJobTestSuite) TestRunAttributeUpdateError() {
data := suite.createDataRecords(3, 1)
data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{1}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{1}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return([]*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1}}}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessLabelFilter").Return([]*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1}}}, nil).Once()
mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(MockDigest), nil)
execAttrs := make(map[string]interface{})
@ -155,6 +152,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunAttributeUpdateError() {
params := job.Parameters{}
params[export.JobModeKey] = export.JobModeExport
params["JobId"] = JobId
params["Request"] = map[string]interface{}{}
ctx := &mockjobservice.MockJobContext{}
err := suite.job.Run(ctx, params)
@ -199,7 +197,7 @@ func (suite *ScanDataExportJobTestSuite) TestExtractCriteria() {
func (suite *ScanDataExportJobTestSuite) TestRunWithCriteria() {
{
data := suite.createDataRecords(3, 1)
data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
@ -262,7 +260,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteria() {
{
mock.OnAnything(suite.sysArtifactMgr, "Create").Return(int64(1), nil).Once()
data := suite.createDataRecords(3, 1)
data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(MockDigest), nil)
@ -325,7 +323,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteria() {
func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForProjectIdFilter() {
{
data := suite.createDataRecords(3, 1)
data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
@ -338,11 +336,9 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForProjectIdFilter()
execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once()
repoCandidate1 := &selector.Candidate{NamespaceID: 1}
repoCandidates := []*selector.Candidate{repoCandidate1}
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return(nil, errors.New("test error")).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return(repoCandidates, nil)
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(repoCandidates, nil)
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return(nil, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(nil, nil).Once()
criteria := export.Request{
CVEIds: "CVE-123",
@ -378,7 +374,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForProjectIdFilter()
// empty list of projects
{
data := suite.createDataRecords(3, 1)
data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
@ -391,11 +387,9 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForProjectIdFilter()
execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once()
repoCandidate1 := &selector.Candidate{NamespaceID: 1}
repoCandidates := []*selector.Candidate{repoCandidate1}
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return(repoCandidates, nil)
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(repoCandidates, nil)
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{1}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return([]*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1}}}, nil).Once()
criteria := export.Request{
CVEIds: "CVE-123",
@ -420,7 +414,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForProjectIdFilter()
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
})
suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
suite.sysArtifactMgr.AssertNotCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), mock.Anything)
_, err = os.Stat("/tmp/scandata_export_100.csv")
@ -434,7 +428,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForProjectIdFilter()
func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilter() {
{
data := suite.createDataRecords(3, 1)
data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
@ -447,11 +441,9 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilte
execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once()
repoCandidate1 := &selector.Candidate{NamespaceID: 1}
repoCandidates := []*selector.Candidate{repoCandidate1}
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{1}, errors.New("test error")).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return(nil, errors.New("test error"))
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(repoCandidates, nil)
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{1}, errors.New("test error")).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return([]*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1}}}, nil).Once()
criteria := export.Request{
CVEIds: "CVE-123",
@ -487,7 +479,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilte
// empty list of repo ids
{
data := suite.createDataRecords(3, 1)
data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
@ -500,10 +492,9 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilte
execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once()
repoCandidates := make([]*selector.Candidate, 0)
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return(repoCandidates, nil)
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(repoCandidates, nil)
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return([]*artifact.Artifact{}, nil).Once()
criteria := export.Request{
CVEIds: "CVE-123",
@ -528,7 +519,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilte
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
})
suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
suite.sysArtifactMgr.AssertNotCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), mock.Anything)
_, err = os.Stat("/tmp/scandata_export_100.csv")
@ -541,7 +532,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilte
func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithTagFilter() {
{
data := suite.createDataRecords(3, 1)
data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
@ -554,11 +545,9 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithT
execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once()
repoCandidate1 := &selector.Candidate{NamespaceID: 1}
repoCandidates := []*selector.Candidate{repoCandidate1}
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{1}, errors.New("test error")).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return(repoCandidates, nil)
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(nil, errors.New("test error"))
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{1}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(nil, errors.New("test error")).Once()
criteria := export.Request{
CVEIds: "CVE-123",
@ -594,7 +583,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithT
// empty list of repo ids after applying tag filters
{
data := suite.createDataRecords(3, 1)
data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
@ -607,10 +596,9 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithT
execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once()
repoCandidates := make([]*selector.Candidate, 0)
mock.OnAnything(suite.filterProcessor, "ProcessProjectFilter").Return([]int64{}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return(repoCandidates, nil)
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(make([]*selector.Candidate, 0), nil)
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(nil, nil).Once()
criteria := export.Request{
CVEIds: "CVE-123",
@ -635,7 +623,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithT
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
})
suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
suite.sysArtifactMgr.AssertNotCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), mock.Anything)
_, err = os.Stat("/tmp/scandata_export_100.csv")
@ -647,7 +635,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithT
}
func (suite *ScanDataExportJobTestSuite) TestExportDigestCalculationErrorsOut() {
data := suite.createDataRecords(3, 1)
data := suite.createDataRecords(3)
mock.OnAnything(suite.exportMgr, "Fetch").Return(data, nil).Once()
mock.OnAnything(suite.exportMgr, "Fetch").Return(make([]export.Data, 0), nil).Once()
mock.OnAnything(suite.digestCalculator, "Calculate").Return(digest.Digest(""), errors.New("test error"))
@ -677,7 +665,7 @@ func (suite *ScanDataExportJobTestSuite) TearDownTest() {
suite.NoError(err)
}
func (suite *ScanDataExportJobTestSuite) createDataRecords(numRecs int, ownerId int64) []export.Data {
func (suite *ScanDataExportJobTestSuite) createDataRecords(numRecs int) []export.Data {
data := make([]export.Data, 0)
for i := 1; i <= numRecs; i++ {
dataRec := export.Data{

21
src/pkg/scan/export/constants.go
View File

@ -4,14 +4,15 @@ package export
type CsvJobVendorID string
const (
ProjectIDsAttribute = "project_ids"
JobNameAttribute = "job_name"
UserNameAttribute = "user_name"
ScanDataExportDir = "/var/scandata_exports"
QueryPageSize = 100000
ArtifactGroupSize = 10000
DigestKey = "artifact_digest"
CreateTimestampKey = "create_ts"
Vendor = "SCAN_DATA_EXPORT"
CsvJobVendorIDKey = CsvJobVendorID("vendorId")
ProjectIDsAttribute = "project_ids"
JobNameAttribute = "job_name"
UserNameAttribute = "user_name"
StatusMessageAttribute = "status_message"
ScanDataExportDir = "/var/scandata_exports"
QueryPageSize = 100000
ArtifactGroupSize = 10000
DigestKey = "artifact_digest"
CreateTimestampKey = "create_ts"
Vendor = "SCAN_DATA_EXPORT"
CsvJobVendorIDKey = CsvJobVendorID("vendorId")
)

14
src/server/v2.0/handler/scanexport.go
View File

@ -160,13 +160,6 @@ func (se *scanDataExportAPI) DownloadScanData(ctx context.Context, params operat
return se.SendError(ctx, err)
}
// check if the CSV artifact for the execution exists
if !execution.FilePresent {
return middleware.ResponderFunc(func(writer http.ResponseWriter, producer runtime.Producer) {
writer.WriteHeader(http.StatusNotFound)
})
}
// check if the execution being downloaded is owned by the current user
secContext, err := se.GetSecurityContext(ctx)
if err != nil {
@ -179,6 +172,13 @@ func (se *scanDataExportAPI) DownloadScanData(ctx context.Context, params operat
})
}
// check if the CSV artifact for the execution exists
if !execution.FilePresent {
return middleware.ResponderFunc(func(writer http.ResponseWriter, producer runtime.Producer) {
writer.WriteHeader(http.StatusNotFound)
})
}
repositoryName := fmt.Sprintf("scandata_export_%v", params.ExecutionID)
file, err := se.sysArtifactMgr.Read(ctx, strings.ToLower(export.Vendor), repositoryName, execution.ExportDataDigest)
if err != nil {

2
src/server/v2.0/handler/scanexport_test.go
View File

@ -514,7 +514,7 @@ func (suite *ScanExportTestSuite) TestDownloadScanDataNoCsvFilePresent() {
StartTime: startTime,
EndTime: endTime,
ExportDataDigest: "datadigest",
UserName: "test-user",
UserName: "test-user1",
FilePresent: false,
}
mock.OnAnything(suite.scanExportCtl, "GetExecution").Return(execution, nil)

35
tests/apitests/python/test_scan_data_export.py
View File

@ -14,6 +14,7 @@ from library.artifact import Artifact
from library.scan import Scan
from library.repository import push_self_build_image_to_project
class TestScanDataExport(unittest.TestCase):
@suppress_urllib3_warning
@ -45,25 +46,26 @@ class TestScanDataExport(unittest.TestCase):
11. Verify that the export scan data execution triggered by the user (UA) cannot be download by other users;
12. User (UA) should be able to download the triggered export scan data execution
13. Verify that the downloaded export scan data execution cannot be downloaded again
14. Verify the status message if no cve found or matched
"""
url = ADMIN_CLIENT["endpoint"]
user_password = "Aa123456"
# 1. Create user(UA)
user_id, user_name = self.user.create_user(user_password = user_password, **ADMIN_CLIENT)
user_client = dict(endpoint = url, username = user_name, password = user_password)
user_id, user_name = self.user.create_user(user_password=user_password, **ADMIN_CLIENT)
user_client = dict(endpoint=url, username=user_name, password=user_password)
# 2.1. Create private project(PA) by user(UA)
project_id, project_name = self.project.create_project(metadata = {"public": "false"}, **user_client)
project_id, project_name = self.project.create_project(metadata={"public": "false"}, **user_client)
# 2.2. Get private project of uesr-001, uesr-001 can see only one private project which is project-001
self.project.projects_should_exist(dict(public=False), expected_count = 1, expected_project_id = project_id, **user_client)
self.project.projects_should_exist(dict(public=False), expected_count=1, expected_project_id=project_id, **user_client)
# 3. Push a new image(IA) in project(PA) by user(UA)
push_self_build_image_to_project(project_name, harbor_server, user_name, user_password, self.image, self.tag)
# 4. Send scan image command and get tag(TA) information to check scan result, it should be finished
self.scan.scan_artifact(project_name, self.image, self.tag, **user_client)
self.artifact.check_image_scan_result(project_name, self.image, self.tag, with_scan_overview = True, **user_client)
self.artifact.check_image_scan_result(project_name, self.image, self.tag, with_scan_overview=True, **user_client)
# 5. Verify trigger export scan data execution but does not specify Scan-Data-Type status code should be 422
self.scan_data_export.export_scan_data("", projects=[project_id], expect_status_code=422, expect_response_body="X-Scan-Data-Type in header is required")
@ -90,26 +92,31 @@ class TestScanDataExport(unittest.TestCase):
self.assertEqual(user_name, execution_list.items[0].user_name)
# 10. Wait for the export scan data execution to succeed
executio_status = None
execution = None
for i in range(5):
print("wait for the job to finish:", i)
execution = self.scan_data_export.get_scan_data_export_execution(execution_id, **user_client)
executio_status = execution.status
if executio_status == "Success":
if execution.status == "Success":
self.assertEqual(user_name, execution.user_name)
self.assertEqual(user_id, execution.user_id)
break
time.sleep(2)
self.assertEqual(executio_status, "Success")
self.assertEqual(execution.status, "Success")
# 11. Verify that the export scan data execution triggered by the user (UA) cannot be download by other users
self.scan_data_export.download_scan_data(execution_id, expect_status_code=403)
# 12. User (UA) should be able to download the triggered export scan data execution
self.scan_data_export.download_scan_data(execution_id, **user_client)
# The csv file will not be able to downloaded if it is empty, so only check download if the file is present, otherwise check the status message
if execution.file_present:
# 12. User (UA) should be able to download the triggered export scan data execution
self.scan_data_export.download_scan_data(execution_id, **user_client)
# 13. Verify that the downloaded export scan data execution cannot be downloaded again
self.scan_data_export.download_scan_data(execution_id, expect_status_code=404, **user_client)
else:
# 14. Verify the status message if no cve found or matched
self.assertEqual("No vulnerabilities found or matched", execution.status_text)
# 13. Verify that the downloaded export scan data execution cannot be downloaded again
self.scan_data_export.download_scan_data(execution_id, expect_status_code=404, **user_client)
if __name__ == '__main__':
unittest.main()
unittest.main()