Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-04 17:49:48 +01:00
Code Issues Projects Releases Wiki Activity

[cherry-pick] fix: fix the invalid jobid for scan data export (#18420)

Browse Source 
fix: fix the invalid jobid for scan data export

Change the JobId param type from int to string, use int will bring some
problems for encode/decode type mismatch which generate the invalid
repository name.

Fixes: #18380

Signed-off-by: chlins <chenyuzh@vmware.com>
This commit is contained in:
Chlins Zhang 2023-03-28 17:55:06 +08:00 committed by GitHub
parent 77903fafc2
commit 4a09c55735
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 71 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/controller/scandataexport/execution.go
View File

@ -128,8 +128,8 @@ func (c *controller) Start(ctx context.Context, request export.Request) (executi
// create a job object and fill with metadata and parameters // create a job object and fill with metadata and parameters
params := make(map[string]interface{}) params := make(map[string]interface{})
params["JobId"] = id params[export.JobID] = fmt.Sprintf("%d", id)
params["Request"] = request params[export.JobRequest] = request
params[export.JobModeKey] = export.JobModeExport params[export.JobModeKey] = export.JobModeExport
j := &task.Job{ j := &task.Job{

47
src/jobservice/job/impl/scandataexport/scan_data_export.go
View File

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"strconv"
"strings" "strings"
"github.com/gocarina/gocsv" "github.com/gocarina/gocsv"
@ -47,7 +48,8 @@ func (sde *ScanDataExport) MaxCurrency() uint {
// still less that the number declared by the method 'MaxFails'. // still less that the number declared by the method 'MaxFails'.
// //
// Returns: // Returns:
// true for retry and false for none-retry //
// true for retry and false for none-retry
func (sde *ScanDataExport) ShouldRetry() bool { func (sde *ScanDataExport) ShouldRetry() bool {
return true return true
} }
@ -66,8 +68,8 @@ func (sde *ScanDataExport) Validate(params job.Parameters) error {
// params map[string]interface{} : parameters with key-pair style for the job execution. // params map[string]interface{} : parameters with key-pair style for the job execution.
// //
// Returns: // Returns:
// error if failed to run. NOTES: If job is stopped or cancelled, a specified error should be returned
// //
// error if failed to run. NOTES: If job is stopped or cancelled, a specified error should be returned
func (sde *ScanDataExport) Run(ctx job.Context, params job.Parameters) error { func (sde *ScanDataExport) Run(ctx job.Context, params job.Parameters) error {
if _, ok := params[export.JobModeKey]; !ok { if _, ok := params[export.JobModeKey]; !ok {
return errors.Errorf("no mode specified for scan data export execution") return errors.Errorf("no mode specified for scan data export execution")
@ -77,7 +79,7 @@ func (sde *ScanDataExport) Run(ctx job.Context, params job.Parameters) error {
logger := ctx.GetLogger() logger := ctx.GetLogger()
logger.Infof("Scan data export job started in mode : %v", mode) logger.Infof("Scan data export job started in mode : %v", mode)
sde.init() sde.init()
fileName := fmt.Sprintf("%s/scandata_export_%v.csv", sde.scanDataExportDirPath, params["JobId"]) fileName := fmt.Sprintf("%s/scandata_export_%s.csv", sde.scanDataExportDirPath, params[export.JobID])
// ensure that CSV files are cleared post the completion of the Run. // ensure that CSV files are cleared post the completion of the Run.
defer sde.cleanupCsvFile(ctx, fileName, params) defer sde.cleanupCsvFile(ctx, fileName, params)
@ -92,12 +94,12 @@ func (sde *ScanDataExport) Run(ctx job.Context, params job.Parameters) error {
logger.Errorf("Error when calculating checksum for generated file: %v", err) logger.Errorf("Error when calculating checksum for generated file: %v", err)
return err return err
} }
logger.Infof("Export Job Id = %v, FileName = %s, Hash = %v", params["JobId"], fileName, hash) logger.Infof("Export Job Id = %s, FileName = %s, Hash = %v", params[export.JobID], fileName, hash)
csvFile, err := os.OpenFile(fileName, os.O_RDONLY, os.ModePerm) csvFile, err := os.OpenFile(fileName, os.O_RDONLY, os.ModePerm)
if err != nil { if err != nil {
logger.Errorf( logger.Errorf(
"Export Job Id = %v. Error when moving report file %s to persistent storage: %v", params["JobId"], fileName, err) "Export Job Id = %s. Error when moving report file %s to persistent storage: %v", params[export.JobID], fileName, err)
return err return err
} }
baseFileName := filepath.Base(fileName) baseFileName := filepath.Base(fileName)
@ -108,12 +110,12 @@ func (sde *ScanDataExport) Run(ctx job.Context, params job.Parameters) error {
logger.Errorf("Error when fetching file size: %v", err) logger.Errorf("Error when fetching file size: %v", err)
return err return err
} }
logger.Infof("Export Job Id = %v. CSV file size: %d", params["JobId"], stat.Size()) logger.Infof("Export Job Id = %s. CSV file size: %d", params[export.JobID], stat.Size())
csvExportArtifactRecord := model.SystemArtifact{Repository: repositoryName, Digest: hash.String(), Size: stat.Size(), Type: "ScanData_CSV", Vendor: strings.ToLower(export.Vendor)} csvExportArtifactRecord := model.SystemArtifact{Repository: repositoryName, Digest: hash.String(), Size: stat.Size(), Type: "ScanData_CSV", Vendor: strings.ToLower(export.Vendor)}
artID, err := sde.sysArtifactMgr.Create(ctx.SystemContext(), &csvExportArtifactRecord, csvFile) artID, err := sde.sysArtifactMgr.Create(ctx.SystemContext(), &csvExportArtifactRecord, csvFile)
if err != nil { if err != nil {
logger.Errorf( logger.Errorf(
"Export Job Id = %v. Error when persisting report file %s to persistent storage: %v", params["JobId"], fileName, err) "Export Job Id = %s. Error when persisting report file %s to persistent storage: %v", params[export.JobID], fileName, err)
// NOTICE: this is a tentative solution to resolve error to push empty blob to S3 storage driver, // NOTICE: this is a tentative solution to resolve error to push empty blob to S3 storage driver,
// should unify the behaviour for different drivers. // should unify the behaviour for different drivers.
// Temporary set the status message to extra attributes, then the API handler will fetch it and combined to response for better experience. // Temporary set the status message to extra attributes, then the API handler will fetch it and combined to response for better experience.
@ -123,17 +125,17 @@ func (sde *ScanDataExport) Run(ctx job.Context, params job.Parameters) error {
} }
updateErr := sde.updateExecAttributes(ctx, params, extra) updateErr := sde.updateExecAttributes(ctx, params, extra)
if updateErr != nil { if updateErr != nil {
logger.Errorf("Export Job Id = %v. Error when updating the exec extra attributes 'status_message' to 'No vulnerabilities found or matched': %v", params["JobId"], updateErr) logger.Errorf("Export Job Id = %s. Error when updating the exec extra attributes 'status_message' to 'No vulnerabilities found or matched': %v", params[export.JobID], updateErr)
} }
} }
return err return err
} }
logger.Infof("Export Job Id = %v. Created system artifact: %v for report file %s to persistent storage: %v", params["JobId"], artID, fileName, err) logger.Infof("Export Job Id = %s. Created system artifact: %v for report file %s to persistent storage: %v", params[export.JobID], artID, fileName, err)
err = sde.updateExecAttributes(ctx, params, map[string]interface{}{export.DigestKey: hash.String()}) err = sde.updateExecAttributes(ctx, params, map[string]interface{}{export.DigestKey: hash.String()})
if err != nil { if err != nil {
logger.Errorf("Export Job Id = %v. Error when updating execution record : %v", params["JobId"], err) logger.Errorf("Export Job Id = %s. Error when updating execution record : %v", params[export.JobID], err)
return err return err
} }
logger.Info("Scan data export job completed") logger.Info("Scan data export job completed")
@ -142,11 +144,16 @@ func (sde *ScanDataExport) Run(ctx job.Context, params job.Parameters) error {
} }
func (sde *ScanDataExport) updateExecAttributes(ctx job.Context, params job.Parameters, attrs map[string]interface{}) error { func (sde *ScanDataExport) updateExecAttributes(ctx job.Context, params job.Parameters, attrs map[string]interface{}) error {
execID := int64(params["JobId"].(float64))
exec, err := sde.execMgr.Get(ctx.SystemContext(), execID)
logger := ctx.GetLogger() logger := ctx.GetLogger()
execID, err := strconv.ParseInt(params[export.JobID].(string), 10, 64)
if err != nil { if err != nil {
logger.Errorf("Export Job Id = %v. Error when fetching execution record for update : %v", params["JobId"], err) logger.Errorf("Export Job Id = %s. Error when parse execution id from params: %v", params[export.JobID], err)
return err
}
exec, err := sde.execMgr.Get(ctx.SystemContext(), execID)
if err != nil {
logger.Errorf("Export Job Id = %s. Error when fetching execution record for update : %v", params[export.JobID], err)
return err return err
} }
// copy old extra // copy old extra
@ -175,8 +182,8 @@ func (sde *ScanDataExport) writeCsvFile(ctx job.Context, params job.Parameters,
var exportParams export.Params var exportParams export.Params
var artIDGroups [][]int64 var artIDGroups [][]int64
if criteira, ok := params["Request"]; ok { if criteria, ok := params[export.JobRequest]; ok {
logger.Infof("Request for export : %v", criteira) logger.Infof("Request for export : %v", criteria)
filterCriteria, err := sde.extractCriteria(params) filterCriteria, err := sde.extractCriteria(params)
if err != nil { if err != nil {
return err return err
@ -253,7 +260,7 @@ func (sde *ScanDataExport) writeCsvFile(ctx job.Context, params job.Parameters,
logger.Infof("No more data to fetch. Exiting...") logger.Infof("No more data to fetch. Exiting...")
break break
} }
logger.Infof("Export Group Id = %d, Job Id = %v, Page Number = %d, Page Size = %d Num Records = %d", groupID, params["JobId"], exportParams.PageNumber, exportParams.PageSize, len(data)) logger.Infof("Export Group Id = %d, Job Id = %s, Page Number = %d, Page Size = %d Num Records = %d", groupID, params[export.JobID], exportParams.PageNumber, exportParams.PageSize, len(data))
// for the first page write the CSV with the headers // for the first page write the CSV with the headers
if exportParams.PageNumber == 1 && groupID == 0 { if exportParams.PageNumber == 1 && groupID == 0 {
@ -276,9 +283,9 @@ func (sde *ScanDataExport) writeCsvFile(ctx job.Context, params job.Parameters,
} }
func (sde *ScanDataExport) extractCriteria(params job.Parameters) (*export.Request, error) { func (sde *ScanDataExport) extractCriteria(params job.Parameters) (*export.Request, error) {
filterMap, ok := params["Request"].(map[string]interface{}) filterMap, ok := params[export.JobRequest].(map[string]interface{})
if !ok { if !ok {
return nil, errors.Errorf("malformed criteria '%v'", params["Request"]) return nil, errors.Errorf("malformed criteria '%v'", params[export.JobRequest])
} }
jsonData, err := json.Marshal(filterMap) jsonData, err := json.Marshal(filterMap)
if err != nil { if err != nil {
@ -347,12 +354,12 @@ func (sde *ScanDataExport) init() {
func (sde *ScanDataExport) cleanupCsvFile(ctx job.Context, fileName string, params job.Parameters) { func (sde *ScanDataExport) cleanupCsvFile(ctx job.Context, fileName string, params job.Parameters) {
logger := ctx.GetLogger() logger := ctx.GetLogger()
if _, err := os.Stat(fileName); os.IsNotExist(err) { if _, err := os.Stat(fileName); os.IsNotExist(err) {
logger.Infof("Export Job Id = %v, CSV Export File = %s does not exist. Nothing to do", params["JobId"], fileName) logger.Infof("Export Job Id = %s, CSV Export File = %s does not exist. Nothing to do", params[export.JobID], fileName)
return return
} }
err := os.Remove(fileName) err := os.Remove(fileName)
if err != nil { if err != nil {
logger.Errorf("Export Job Id = %d, CSV Export File = %s could not deleted. Error = %v", params["JobId"], fileName, err) logger.Errorf("Export Job Id = %s, CSV Export File = %s could not deleted. Error = %v", params[export.JobID], fileName, err)
return return
} }
} }

77
src/jobservice/job/impl/scandataexport/scan_data_export_test.go
View File

@ -30,8 +30,11 @@ import (
tasktesting "github.com/goharbor/harbor/src/testing/pkg/task" tasktesting "github.com/goharbor/harbor/src/testing/pkg/task"
) )
const JobId = float64(100) const (
const MockDigest = "mockDigest" ExecID int64 = 1000000
JobId = "1000000"
MockDigest = "mockDigest"
)
type ScanDataExportJobTestSuite struct { type ScanDataExportJobTestSuite struct {
htesting.Suite htesting.Suite
@ -81,7 +84,7 @@ func (suite *ScanDataExportJobTestSuite) TestRun() {
execAttrs := make(map[string]interface{}) execAttrs := make(map[string]interface{})
execAttrs[export.JobNameAttribute] = "test-job" execAttrs[export.JobNameAttribute] = "test-job"
execAttrs[export.UserNameAttribute] = "test-user" execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil) mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: ExecID, ExtraAttrs: execAttrs}, nil)
params := job.Parameters{} params := job.Parameters{}
params[export.JobModeKey] = export.JobModeExport params[export.JobModeKey] = export.JobModeExport
@ -94,7 +97,7 @@ func (suite *ScanDataExportJobTestSuite) TestRun() {
err := suite.job.Run(ctx, params) err := suite.job.Run(ctx, params)
suite.NoError(err) suite.NoError(err)
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool { sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest return sa.Repository == "scandata_export_1000000" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
}) })
suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything) suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
@ -106,8 +109,8 @@ func (suite *ScanDataExportJobTestSuite) TestRun() {
_, ok := m[export.CreateTimestampKey] _, ok := m[export.CreateTimestampKey]
return attrsMap[export.DigestKey] == MockDigest && ok && attrsMap[export.JobNameAttribute] == "test-job" && attrsMap[export.UserNameAttribute] == "test-user" return attrsMap[export.DigestKey] == MockDigest && ok && attrsMap[export.JobNameAttribute] == "test-job" && attrsMap[export.UserNameAttribute] == "test-user"
}) })
suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), extraAttrsMatcher) suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, ExecID, extraAttrsMatcher)
_, err = os.Stat("/tmp/scandata_export_100.csv") _, err = os.Stat("/tmp/scandata_export_1000000.csv")
suite.Truef(os.IsNotExist(err), "Expected CSV file to be deleted") suite.Truef(os.IsNotExist(err), "Expected CSV file to be deleted")
} }
@ -130,7 +133,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunCreateSysArtError() {
execAttrs := make(map[string]interface{}) execAttrs := make(map[string]interface{})
execAttrs[export.JobNameAttribute] = "test-job" execAttrs[export.JobNameAttribute] = "test-job"
execAttrs[export.UserNameAttribute] = "test-user" execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil) mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: ExecID, ExtraAttrs: execAttrs}, nil)
params := job.Parameters{} params := job.Parameters{}
params[export.JobModeKey] = export.JobModeExport params[export.JobModeKey] = export.JobModeExport
@ -143,7 +146,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunCreateSysArtError() {
extraAttrsMatcher := testifymock.MatchedBy(func(attrsMap map[string]interface{}) bool { extraAttrsMatcher := testifymock.MatchedBy(func(attrsMap map[string]interface{}) bool {
return attrsMap["status_message"] == "No vulnerabilities found or matched" && attrsMap[export.JobNameAttribute] == "test-job" && attrsMap[export.UserNameAttribute] == "test-user" return attrsMap["status_message"] == "No vulnerabilities found or matched" && attrsMap[export.JobNameAttribute] == "test-job" && attrsMap[export.UserNameAttribute] == "test-user"
}) })
suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), extraAttrsMatcher) suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, ExecID, extraAttrsMatcher)
} }
func (suite *ScanDataExportJobTestSuite) TestRunAttributeUpdateError() { func (suite *ScanDataExportJobTestSuite) TestRunAttributeUpdateError() {
@ -171,7 +174,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunAttributeUpdateError() {
err := suite.job.Run(ctx, params) err := suite.job.Run(ctx, params)
suite.Error(err) suite.Error(err)
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool { sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest return sa.Repository == "scandata_export_1000000" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
}) })
suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything) suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
@ -183,8 +186,8 @@ func (suite *ScanDataExportJobTestSuite) TestRunAttributeUpdateError() {
_, ok := m[export.CreateTimestampKey] _, ok := m[export.CreateTimestampKey]
return attrsMap[export.DigestKey] == MockDigest && ok && attrsMap[export.JobNameAttribute] == "test-job" && attrsMap[export.UserNameAttribute] == "test-user" return attrsMap[export.DigestKey] == MockDigest && ok && attrsMap[export.JobNameAttribute] == "test-job" && attrsMap[export.UserNameAttribute] == "test-user"
}) })
suite.execMgr.AssertNotCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), extraAttrsMatcher) suite.execMgr.AssertNotCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, ExecID, extraAttrsMatcher)
_, err = os.Stat("/tmp/scandata_export_100.csv") _, err = os.Stat("/tmp/scandata_export_1000000.csv")
suite.Truef(os.IsNotExist(err), "Expected CSV file to be deleted") suite.Truef(os.IsNotExist(err), "Expected CSV file to be deleted")
} }
@ -218,7 +221,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteria() {
execAttrs := make(map[string]interface{}) execAttrs := make(map[string]interface{})
execAttrs[export.JobNameAttribute] = "test-job" execAttrs[export.JobNameAttribute] = "test-job"
execAttrs[export.UserNameAttribute] = "test-user" execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once() mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: ExecID, ExtraAttrs: execAttrs}, nil).Once()
repoCandidates := []int64{1} repoCandidates := []int64{1}
artCandidates := []*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1, Digest: "digest1"}}} artCandidates := []*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1, Digest: "digest1"}}}
@ -247,7 +250,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteria() {
err := suite.job.Run(ctx, params) err := suite.job.Run(ctx, params)
suite.NoError(err) suite.NoError(err)
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool { sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest return sa.Repository == "scandata_export_1000000" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
}) })
suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything) suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
@ -259,8 +262,8 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteria() {
_, ok := m[export.CreateTimestampKey] _, ok := m[export.CreateTimestampKey]
return attrsMap[export.DigestKey] == MockDigest && ok return attrsMap[export.DigestKey] == MockDigest && ok
}) })
suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), extraAttrsMatcher) suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, ExecID, extraAttrsMatcher)
_, err = os.Stat("/tmp/scandata_export_100.csv") _, err = os.Stat("/tmp/scandata_export_1000000.csv")
exportParamsMatcher := testifymock.MatchedBy(func(params export.Params) bool { exportParamsMatcher := testifymock.MatchedBy(func(params export.Params) bool {
return reflect.DeepEqual(params.CVEIds, criteria.CVEIds) return reflect.DeepEqual(params.CVEIds, criteria.CVEIds)
@ -282,7 +285,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteria() {
execAttrs := make(map[string]interface{}) execAttrs := make(map[string]interface{})
execAttrs[export.JobNameAttribute] = "test-job" execAttrs[export.JobNameAttribute] = "test-job"
execAttrs[export.UserNameAttribute] = "test-user" execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once() mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: ExecID, ExtraAttrs: execAttrs}, nil).Once()
repoCandidate1 := &selector.Candidate{NamespaceID: 1} repoCandidate1 := &selector.Candidate{NamespaceID: 1}
repoCandidates := []*selector.Candidate{repoCandidate1} repoCandidates := []*selector.Candidate{repoCandidate1}
@ -309,7 +312,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteria() {
err := suite.job.Run(ctx, params) err := suite.job.Run(ctx, params)
suite.NoError(err) suite.NoError(err)
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool { sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest return sa.Repository == "scandata_export_1000000" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
}) })
suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything) suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
m := make(map[string]interface{}) m := make(map[string]interface{})
@ -320,8 +323,8 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteria() {
_, ok := m[export.CreateTimestampKey] _, ok := m[export.CreateTimestampKey]
return attrsMap[export.DigestKey] == MockDigest && ok return attrsMap[export.DigestKey] == MockDigest && ok
}) })
suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), extraAttrsMatcher) suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, ExecID, extraAttrsMatcher)
_, err = os.Stat("/tmp/scandata_export_100.csv") _, err = os.Stat("/tmp/scandata_export_1000000.csv")
exportParamsMatcher := testifymock.MatchedBy(func(params export.Params) bool { exportParamsMatcher := testifymock.MatchedBy(func(params export.Params) bool {
return reflect.DeepEqual(params.CVEIds, criteria.CVEIds) return reflect.DeepEqual(params.CVEIds, criteria.CVEIds)
@ -345,7 +348,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilte
execAttrs := make(map[string]interface{}) execAttrs := make(map[string]interface{})
execAttrs[export.JobNameAttribute] = "test-job" execAttrs[export.JobNameAttribute] = "test-job"
execAttrs[export.UserNameAttribute] = "test-user" execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once() mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: ExecID, ExtraAttrs: execAttrs}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{1}, errors.New("test error")).Once() mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{1}, errors.New("test error")).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return([]*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1}}}, nil).Once() mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return([]*artifact.Artifact{{Artifact: artpkg.Artifact{ID: 1}}}, nil).Once()
@ -371,11 +374,11 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilte
err := suite.job.Run(ctx, params) err := suite.job.Run(ctx, params)
suite.Error(err) suite.Error(err)
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool { sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest return sa.Repository == "scandata_export_1000000" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
}) })
suite.sysArtifactMgr.AssertNotCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything) suite.sysArtifactMgr.AssertNotCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
suite.execMgr.AssertNotCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), mock.Anything) suite.execMgr.AssertNotCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, ExecID, mock.Anything)
_, err = os.Stat("/tmp/scandata_export_100.csv") _, err = os.Stat("/tmp/scandata_export_1000000.csv")
suite.exportMgr.AssertNotCalled(suite.T(), "Fetch", mock.Anything, mock.Anything) suite.exportMgr.AssertNotCalled(suite.T(), "Fetch", mock.Anything, mock.Anything)
@ -395,7 +398,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilte
execAttrs := make(map[string]interface{}) execAttrs := make(map[string]interface{})
execAttrs[export.JobNameAttribute] = "test-job" execAttrs[export.JobNameAttribute] = "test-job"
execAttrs[export.UserNameAttribute] = "test-user" execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once() mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: ExecID, ExtraAttrs: execAttrs}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{}, nil).Once() mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return([]*artifact.Artifact{}, nil).Once() mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return([]*artifact.Artifact{}, nil).Once()
@ -421,11 +424,11 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdFilte
err := suite.job.Run(ctx, params) err := suite.job.Run(ctx, params)
suite.NoError(err) suite.NoError(err)
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool { sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest return sa.Repository == "scandata_export_1000000" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
}) })
suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything) suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), mock.Anything) suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, ExecID, mock.Anything)
_, err = os.Stat("/tmp/scandata_export_100.csv") _, err = os.Stat("/tmp/scandata_export_1000000.csv")
suite.exportMgr.AssertNotCalled(suite.T(), "Fetch", mock.Anything, mock.Anything) suite.exportMgr.AssertNotCalled(suite.T(), "Fetch", mock.Anything, mock.Anything)
@ -447,7 +450,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithT
execAttrs := make(map[string]interface{}) execAttrs := make(map[string]interface{})
execAttrs[export.JobNameAttribute] = "test-job" execAttrs[export.JobNameAttribute] = "test-job"
execAttrs[export.UserNameAttribute] = "test-user" execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once() mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: ExecID, ExtraAttrs: execAttrs}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{1}, nil).Once() mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{1}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(nil, errors.New("test error")).Once() mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(nil, errors.New("test error")).Once()
@ -473,11 +476,11 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithT
err := suite.job.Run(ctx, params) err := suite.job.Run(ctx, params)
suite.Error(err) suite.Error(err)
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool { sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest return sa.Repository == "scandata_export_1000000" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
}) })
suite.sysArtifactMgr.AssertNotCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything) suite.sysArtifactMgr.AssertNotCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
suite.execMgr.AssertNotCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), mock.Anything) suite.execMgr.AssertNotCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, ExecID, mock.Anything)
_, err = os.Stat("/tmp/scandata_export_100.csv") _, err = os.Stat("/tmp/scandata_export_1000000.csv")
suite.exportMgr.AssertNotCalled(suite.T(), "Fetch", mock.Anything, mock.Anything) suite.exportMgr.AssertNotCalled(suite.T(), "Fetch", mock.Anything, mock.Anything)
@ -497,7 +500,7 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithT
execAttrs := make(map[string]interface{}) execAttrs := make(map[string]interface{})
execAttrs[export.JobNameAttribute] = "test-job" execAttrs[export.JobNameAttribute] = "test-job"
execAttrs[export.UserNameAttribute] = "test-user" execAttrs[export.UserNameAttribute] = "test-user"
mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: int64(JobId), ExtraAttrs: execAttrs}, nil).Once() mock.OnAnything(suite.execMgr, "Get").Return(&task.Execution{ID: ExecID, ExtraAttrs: execAttrs}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{}, nil).Once() mock.OnAnything(suite.filterProcessor, "ProcessRepositoryFilter").Return([]int64{}, nil).Once()
mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(nil, nil).Once() mock.OnAnything(suite.filterProcessor, "ProcessTagFilter").Return(nil, nil).Once()
@ -523,11 +526,11 @@ func (suite *ScanDataExportJobTestSuite) TestRunWithCriteriaForRepositoryIdWithT
err := suite.job.Run(ctx, params) err := suite.job.Run(ctx, params)
suite.NoError(err) suite.NoError(err)
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool { sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest return sa.Repository == "scandata_export_1000000" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
}) })
suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything) suite.sysArtifactMgr.AssertCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, int64(JobId), mock.Anything) suite.execMgr.AssertCalled(suite.T(), "UpdateExtraAttrs", mock.Anything, ExecID, mock.Anything)
_, err = os.Stat("/tmp/scandata_export_100.csv") _, err = os.Stat("/tmp/scandata_export_1000000.csv")
suite.exportMgr.AssertNotCalled(suite.T(), "Fetch", mock.Anything, mock.Anything) suite.exportMgr.AssertNotCalled(suite.T(), "Fetch", mock.Anything, mock.Anything)
@ -549,11 +552,11 @@ func (suite *ScanDataExportJobTestSuite) TestExportDigestCalculationErrorsOut()
err := suite.job.Run(ctx, params) err := suite.job.Run(ctx, params)
suite.Error(err) suite.Error(err)
sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool { sysArtifactRecordMatcher := testifymock.MatchedBy(func(sa *model.SystemArtifact) bool {
return sa.Repository == "scandata_export_100" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest return sa.Repository == "scandata_export_1000000" && sa.Vendor == strings.ToLower(export.Vendor) && sa.Digest == MockDigest
}) })
suite.sysArtifactMgr.AssertNotCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything) suite.sysArtifactMgr.AssertNotCalled(suite.T(), "Create", mock.Anything, sysArtifactRecordMatcher, mock.Anything)
suite.execMgr.AssertNotCalled(suite.T(), "UpdateExtraAttrs") suite.execMgr.AssertNotCalled(suite.T(), "UpdateExtraAttrs")
_, err = os.Stat("/tmp/scandata_export_100.csv") _, err = os.Stat("/tmp/scandata_export_1000000.csv")
suite.Truef(os.IsNotExist(err), "Expected CSV file to be deleted") suite.Truef(os.IsNotExist(err), "Expected CSV file to be deleted")
} }

2
src/pkg/scan/export/manager.go
View File

@ -56,6 +56,8 @@ group by
` `
JobModeExport = "export" JobModeExport = "export"
JobModeKey = "mode" JobModeKey = "mode"
JobID = "JobId"
JobRequest = "Request"
) )
var ( var (