Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-22 08:38:03 +01:00
Code Issues Projects Releases Wiki Activity

add scan report CRUD supporting and

Browse Source 
- change error collection in scan job
- add dead client checking in client pool
- change key word type to interface{} for q.Query
- update bearer authorizer
- add required UT cases

Signed-off-by: Steven Zou <szou@vmware.com>
This commit is contained in:
Steven Zou 2019-09-23 16:03:42 +08:00
parent 0c19eba8c2
commit d616bc3509
16 changed files with 814 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
make/migrations/postgresql/0011_1.10.0_schema.up.sql
View File

@ -19,12 +19,14 @@ CREATE TABLE scanner_registration
CREATE TABLE scan_report
(
id SERIAL PRIMARY KEY NOT NULL,
uuid VARCHAR(64) UNIQUE NOT NULL,
digest VARCHAR(256) NOT NULL,
registration_uuid VARCHAR(64) NOT NULL,
mime_type VARCHAR(256) NOT NULL,
job_id VARCHAR(32),
status VARCHAR(16) NOT NULL,
status_code INTEGER DEFAULT 0,
status_rev BIGINT DEFAULT 0,
report JSON,
start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
end_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,

4
src/core/api/scanners.go
View File

@ -76,7 +76,7 @@ func (sa *ScannerAPI) List() {
}
// Get query key words
kws := make(map[string]string)
kws := make(map[string]interface{})
properties := []string{"name", "description", "url"}
for _, k := range properties {
kw := sa.GetString(k)
@ -316,7 +316,7 @@ func (sa *ScannerAPI) get() *scanner.Registration {
func (sa *ScannerAPI) checkDuplicated(property, value string) bool {
// Explicitly check if conflict
kw := make(map[string]string)
kw := make(map[string]interface{})
kw[property] = value
query := &q.Query{

4
src/core/api/scanners_test.go
View File

@ -296,7 +296,7 @@ func (suite *ScannerAPITestSuite) TestScannerAPIProjectScanner() {
}
func (suite *ScannerAPITestSuite) mockQuery(r *scanner.Registration) {
kw := make(map[string]string, 1)
kw := make(map[string]interface{}, 1)
kw["name"] = r.Name
query := &q.Query{
Keywords: kw,
@ -304,7 +304,7 @@ func (suite *ScannerAPITestSuite) mockQuery(r *scanner.Registration) {
emptyL := make([]*scanner.Registration, 0)
suite.mockC.On("ListRegistrations", query).Return(emptyL, nil)
kw2 := make(map[string]string, 1)
kw2 := make(map[string]interface{}, 1)
kw2["url"] = r.URL
query2 := &q.Query{
Keywords: kw2,

2
src/pkg/q/query.go
View File

@ -21,5 +21,5 @@ type Query struct {
// Page size
PageSize int64
// List of key words
Keywords map[string]string
Keywords map[string]interface{}
}

140
src/pkg/scan/dao/scan/report.go Normal file
View File

@ -0,0 +1,140 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"fmt"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/pkg/errors"
)
func init() {
orm.RegisterModel(new(Report))
}
// CreateReport creates new report
func CreateReport(r *Report) (int64, error) {
o := dao.GetOrmer()
return o.Insert(r)
}
// DeleteReport deletes the given report
func DeleteReport(uuid string) error {
o := dao.GetOrmer()
qt := o.QueryTable(new(Report))
// Delete report with query way
count, err := qt.Filter("uuid", uuid).Delete()
if err != nil {
return err
}
if count == 0 {
return errors.Errorf("no report with uuid %s deleted", uuid)
}
return nil
}
// ListReports lists the reports with given query parameters.
// Keywords in query here will be enforced with `exact` way.
func ListReports(query *q.Query) ([]*Report, error) {
o := dao.GetOrmer()
qt := o.QueryTable(new(Report))
if query != nil {
if len(query.Keywords) > 0 {
for k, v := range query.Keywords {
if vv, ok := v.([]interface{}); ok {
qt = qt.Filter(fmt.Sprintf("%s__in", k), vv...)
}
qt = qt.Filter(k, v)
}
}
if query.PageNumber > 0 && query.PageSize > 0 {
qt = qt.Limit(query.PageSize, (query.PageNumber-1)*query.PageSize)
}
}
l := make([]*Report, 0)
_, err := qt.All(&l)
return l, err
}
// UpdateReportData only updates the `report` column with conditions matched.
func UpdateReportData(uuid string, report string, statusRev int64) error {
o := dao.GetOrmer()
qt := o.QueryTable(new(Report))
data := make(orm.Params)
data["report"] = report
data["status_rev"] = statusRev
count, err := qt.Filter("uuid", uuid).
Filter("status_rev__lte", statusRev).Update(data)
if err != nil {
return err
}
if count == 0 {
return errors.Errorf("no report with uuid %s updated", uuid)
}
return nil
}
// UpdateReportStatus updates the report `status` with conditions matched.
func UpdateReportStatus(uuid string, status string, statusCode int, statusRev int64) error {
o := dao.GetOrmer()
qt := o.QueryTable(new(Report))
data := make(orm.Params)
data["status"] = status
data["status_code"] = statusCode
data["status_rev"] = statusRev
count, err := qt.Filter("uuid", uuid).
Filter("status_rev__lte", statusRev).
Filter("status_code__lte", statusCode).Update(data)
if err != nil {
return err
}
if count == 0 {
return errors.Errorf("no report with uuid %s updated", uuid)
}
return nil
}
// UpdateJobID updates the report `job_id` column
func UpdateJobID(uuid string, jobID string) error {
o := dao.GetOrmer()
qt := o.QueryTable(new(Report))
params := make(orm.Params, 1)
params["job_id"] = jobID
_, err := qt.Filter("uuid", uuid).Update(params)
return err
}

131
src/pkg/scan/dao/scan/report_test.go Normal file
View File

@ -0,0 +1,131 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"testing"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/q"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
// ReportTestSuite is test suite of testing report DAO.
type ReportTestSuite struct {
suite.Suite
}
// TestReport is the entry of ReportTestSuite.
func TestReport(t *testing.T) {
suite.Run(t, &ReportTestSuite{})
}
// SetupSuite prepares env for test suite.
func (suite *ReportTestSuite) SetupSuite() {
dao.PrepareTestForPostgresSQL()
}
// SetupTest prepares env for test case.
func (suite *ReportTestSuite) SetupTest() {
r := &Report{
UUID: "uuid",
Digest: "digest1001",
RegistrationUUID: "ruuid",
MimeType: v1.MimeTypeNativeReport,
Status: job.PendingStatus.String(),
StatusCode: job.PendingStatus.Code(),
}
id, err := CreateReport(r)
require.NoError(suite.T(), err)
require.Condition(suite.T(), func() (success bool) {
success = id > 0
return
})
}
// TearDownTest clears enf for test case.
func (suite *ReportTestSuite) TearDownTest() {
err := DeleteReport("uuid")
require.NoError(suite.T(), err)
}
// TestReportList tests list reports with query parameters.
func (suite *ReportTestSuite) TestReportList() {
query1 := &q.Query{
PageSize: 1,
PageNumber: 1,
Keywords: map[string]interface{}{
"digest": "digest1001",
"registration_uuid": "ruuid",
"mime_type": v1.MimeTypeNativeReport,
},
}
l, err := ListReports(query1)
require.NoError(suite.T(), err)
require.Equal(suite.T(), 1, len(l))
query2 := &q.Query{
PageSize: 1,
PageNumber: 1,
Keywords: map[string]interface{}{
"digest": "digest1002",
},
}
l, err = ListReports(query2)
require.NoError(suite.T(), err)
require.Equal(suite.T(), 0, len(l))
}
// TestReportUpdateJobID tests update job ID of the report.
func (suite *ReportTestSuite) TestReportUpdateJobID() {
err := UpdateJobID("uuid", "jobid001")
require.NoError(suite.T(), err)
l, err := ListReports(nil)
require.NoError(suite.T(), err)
require.Equal(suite.T(), 1, len(l))
assert.Equal(suite.T(), "jobid001", l[0].JobID)
}
// TestReportUpdateReportData tests update the report data.
func (suite *ReportTestSuite) TestReportUpdateReportData() {
err := UpdateReportData("uuid", "{}", 1000)
require.NoError(suite.T(), err)
l, err := ListReports(nil)
require.NoError(suite.T(), err)
require.Equal(suite.T(), 1, len(l))
assert.Equal(suite.T(), "{}", l[0].Report)
err = UpdateReportData("uuid", "{\"a\": 900}", 900)
require.Error(suite.T(), err)
}
// TestReportUpdateStatus tests update the report status.
func (suite *ReportTestSuite) TestReportUpdateStatus() {
err := UpdateReportStatus("uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 1000)
require.NoError(suite.T(), err)
err = UpdateReportStatus("uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 900)
require.Error(suite.T(), err)
err = UpdateReportStatus("uuid", job.PendingStatus.String(), job.PendingStatus.Code(), 1000)
require.Error(suite.T(), err)
}

2
src/pkg/scan/dao/scanner/registration_test.go
View File

@ -107,7 +107,7 @@ func (suite *RegistrationDAOTestSuite) TestList() {
require.Equal(suite.T(), 1, len(l))
// with query and found items
keywords := make(map[string]string)
keywords := make(map[string]interface{})
keywords["description"] = "sample"
l, err = ListRegistrations(&q.Query{
PageSize: 5,

62
src/pkg/scan/job.go
View File

@ -129,43 +129,15 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error {
return errors.Wrap(err, "run scan job")
}
// Exit signal
exit := make(chan bool, 1)
// Done signal
done := make(chan bool, 1)
// Collect errors
eChan := make(chan error, 1)
go func() {
defer func() {
// Done!
done <- true
}()
for {
select {
case e := <-eChan:
if err != nil {
err = errors.Wrap(e, err.Error())
} else {
err = e
}
case <-exit:
// Gracefully exit
return
case <-ctx.SystemContext().Done():
// Terminated by system
return
}
}
}()
// For collecting errors
errs := make([]error, len(mimes))
// Concurrently retrieving report by different mime types
wg := &sync.WaitGroup{}
wg.Add(len(mimes))
for _, mt := range mimes {
go func(m string) {
for i, mt := range mimes {
go func(i int, m string) {
defer wg.Done()
// Log info
@ -191,13 +163,13 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error {
continue
}
eChan <- errors.Wrap(err, fmt.Sprintf("check scan report with mime type %s", m))
errs[i] = errors.Wrap(err, fmt.Sprintf("check scan report with mime type %s", m))
return
}
// Make sure the data is aligned with the v1 spec.
if _, err = report.ResolveData(m, []byte(rawReport)); err != nil {
eChan <- errors.Wrap(err, "scan job: resolve report data")
errs[i] = errors.Wrap(err, "scan job: resolve report data")
return
}
@ -222,25 +194,33 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error {
}
// Send error and exit
eChan <- errors.Wrap(er, fmt.Sprintf("check in scan report for mime type %s", m))
errs[i] = errors.Wrap(er, fmt.Sprintf("check in scan report for mime type %s", m))
return
case <-ctx.SystemContext().Done():
// Terminated by system
return
case <-time.After(checkTimeout):
eChan <- errors.New("check scan report timeout")
errs[i] = errors.New("check scan report timeout")
return
}
}
}(mt)
}(i, mt)
}
// Wait for all the retrieving routines are completed
wg.Wait()
// Stop error collection goroutine
exit <- true
// done!
<-done
// Merge errors
for _, e := range errs {
if e != nil {
if err != nil {
err = errors.Wrap(e, err.Error())
} else {
err = e
}
}
}
// Log error to the job log
if err != nil {
myLogger.Error(err)

169
src/pkg/scan/report/base_manager.go Normal file
View File

@ -0,0 +1,169 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package report
import (
"time"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/google/uuid"
"github.com/pkg/errors"
)
// basicManager is a default implementation of report manager.
type basicManager struct{}
// NewManager news basic manager.
func NewManager() Manager {
return &basicManager{}
}
// Create ...
func (bm *basicManager) Create(r *scan.Report) (string, error) {
// Validate report object
if r == nil {
return "", errors.New("nil scan report object")
}
if len(r.Digest) == 0 || len(r.RegistrationUUID) == 0 || len(r.MimeType) == 0 {
return "", errors.New("malformed scan report object")
}
// Check if there is existing report copy
// Limit only one scanning performed by a given provider on the specified artifact can be there
kws := make(map[string]interface{}, 3)
kws["digest"] = r.Digest
kws["registration_uuid"] = r.RegistrationUUID
kws["mime_type"] = []interface{}{r.MimeType}
existingCopies, err := scan.ListReports(&q.Query{
PageNumber: 1,
PageSize: 1,
Keywords: kws,
})
if err != nil {
return "", errors.Wrap(err, "check existence of report")
}
// Delete existing copy
if len(existingCopies) > 0 {
theCopy := existingCopies[0]
// Status conflict
theStatus := job.Status(theCopy.Status)
if theStatus.Compare(job.RunningStatus) <= 0 {
return "", errors.Errorf("conflict: a previous scanning is %s", theCopy.Status)
}
// Otherwise it will be a completed report
// Clear it before insert this new one
if err := scan.DeleteReport(theCopy.UUID); err != nil {
return "", errors.Wrap(err, "clear old scan report")
}
}
// Assign uuid
UUID, err := uuid.NewUUID()
if err != nil {
return "", errors.Wrap(err, "create report: new UUID")
}
r.UUID = UUID.String()
// Fill in / override the related properties
r.StartTime = time.Now().UTC()
r.Status = job.PendingStatus.String()
r.StatusCode = job.PendingStatus.Code()
// Insert
if _, err = scan.CreateReport(r); err != nil {
return "", errors.Wrap(err, "create report")
}
return r.UUID, nil
}
// GetBy ...
func (bm *basicManager) GetBy(digest string, registrationUUID string, mimeTypes []string) ([]*scan.Report, error) {
if len(digest) == 0 {
return nil, errors.New("empty digest to get report data")
}
kws := make(map[string]interface{})
kws["digest"] = digest
if len(registrationUUID) > 0 {
kws["registration_uuid"] = registrationUUID
}
if len(mimeTypes) > 0 {
kws["mime_type"] = mimeTypes
}
// Query all
query := &q.Query{
PageNumber: 0,
Keywords: kws,
}
return scan.ListReports(query)
}
// UpdateScanJobID ...
func (bm *basicManager) UpdateScanJobID(uuid string, jobID string) error {
if len(uuid) == 0 || len(jobID) == 0 {
return errors.New("bad arguments")
}
return scan.UpdateJobID(uuid, jobID)
}
// UpdateStatus ...
func (bm *basicManager) UpdateStatus(uuid string, status string, rev int64) error {
if len(uuid) == 0 {
return errors.New("missing uuid")
}
if rev <= 0 {
return errors.New("invalid data revision")
}
stCode := job.ErrorStatus.Code()
st := job.Status(status)
// Check if it is job valid status.
// Probably an error happened before submitting jobs.
if st.Code() != -1 {
// Assign error code
stCode = st.Code()
}
return scan.UpdateReportStatus(uuid, status, stCode, rev)
}
// UpdateReportData ...
func (bm *basicManager) UpdateReportData(uuid string, report string, rev int64) error {
if len(uuid) == 0 {
return errors.New("missing uuid")
}
if rev <= 0 {
return errors.New("invalid data revision")
}
if len(report) == 0 {
return errors.New("missing report JSON data")
}
return scan.UpdateReportData(uuid, report, rev)
}

156
src/pkg/scan/report/base_manager_test.go Normal file
View File

@ -0,0 +1,156 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package report
import (
"testing"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
// TestManagerSuite is a test suite for the report manager.
type TestManagerSuite struct {
suite.Suite
m Manager
rpUUID string
}
// TestManager is an entry of suite TestManagerSuite.
func TestManager(t *testing.T) {
suite.Run(t, &TestManagerSuite{})
}
// SetupSuite prepares test env for suite TestManagerSuite.
func (suite *TestManagerSuite) SetupSuite() {
dao.PrepareTestForPostgresSQL()
suite.m = NewManager()
}
// SetupTest prepares env for test cases.
func (suite *TestManagerSuite) SetupTest() {
rp := &scan.Report{
Digest: "d1000",
RegistrationUUID: "ruuid",
MimeType: v1.MimeTypeNativeReport,
}
uuid, err := suite.m.Create(rp)
require.NoError(suite.T(), err)
require.NotEmpty(suite.T(), uuid)
suite.rpUUID = uuid
}
// TearDownTest clears test env for test cases.
func (suite *TestManagerSuite) TearDownTest() {
// No delete method defined in manager as no requirement,
// so, to clear env, call dao method here
err := scan.DeleteReport(suite.rpUUID)
require.NoError(suite.T(), err)
}
// TestManagerCreateWithExisting tests the case that a copy already is there when creating report.
func (suite *TestManagerSuite) TestManagerCreateWithExisting() {
err := suite.m.UpdateStatus(suite.rpUUID, job.SuccessStatus.String(), 2000)
require.NoError(suite.T(), err)
rp := &scan.Report{
Digest: "d1000",
RegistrationUUID: "ruuid",
MimeType: v1.MimeTypeNativeReport,
}
uuid, err := suite.m.Create(rp)
require.NoError(suite.T(), err)
require.NotEmpty(suite.T(), uuid)
assert.NotEqual(suite.T(), suite.rpUUID, uuid)
suite.rpUUID = uuid
}
// TestManagerGetBy tests the get by method.
func (suite *TestManagerSuite) TestManagerGetBy() {
l, err := suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport})
require.NoError(suite.T(), err)
require.Equal(suite.T(), 1, len(l))
assert.Equal(suite.T(), suite.rpUUID, l[0].UUID)
l, err = suite.m.GetBy("d1000", "ruuid", nil)
require.NoError(suite.T(), err)
require.Equal(suite.T(), 1, len(l))
assert.Equal(suite.T(), suite.rpUUID, l[0].UUID)
l, err = suite.m.GetBy("d1000", "", nil)
require.NoError(suite.T(), err)
require.Equal(suite.T(), 1, len(l))
assert.Equal(suite.T(), suite.rpUUID, l[0].UUID)
}
// TestManagerUpdateJobID tests update job ID method.
func (suite *TestManagerSuite) TestManagerUpdateJobID() {
l, err := suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport})
require.NoError(suite.T(), err)
require.Equal(suite.T(), 1, len(l))
oldJID := l[0].JobID
err = suite.m.UpdateScanJobID(suite.rpUUID, "jID1001")
require.NoError(suite.T(), err)
l, err = suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport})
require.NoError(suite.T(), err)
require.Equal(suite.T(), 1, len(l))
assert.NotEqual(suite.T(), oldJID, l[0].JobID)
assert.Equal(suite.T(), "jID1001", l[0].JobID)
}
// TestManagerUpdateStatus tests update status method
func (suite *TestManagerSuite) TestManagerUpdateStatus() {
l, err := suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport})
require.NoError(suite.T(), err)
require.Equal(suite.T(), 1, len(l))
oldSt := l[0].Status
err = suite.m.UpdateStatus(suite.rpUUID, job.SuccessStatus.String(), 10000)
require.NoError(suite.T(), err)
l, err = suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport})
require.NoError(suite.T(), err)
require.Equal(suite.T(), 1, len(l))
assert.NotEqual(suite.T(), oldSt, l[0].Status)
assert.Equal(suite.T(), job.SuccessStatus.String(), l[0].Status)
}
// TestManagerUpdateReportData tests update job report data.
func (suite *TestManagerSuite) TestManagerUpdateReportData() {
err := suite.m.UpdateReportData(suite.rpUUID, "{\"a\":1000}", 1000)
require.NoError(suite.T(), err)
l, err := suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport})
require.NoError(suite.T(), err)
require.Equal(suite.T(), 1, len(l))
assert.Equal(suite.T(), "{\"a\":1000}", l[0].Report)
}

80
src/pkg/scan/report/manager.go Normal file
View File

@ -0,0 +1,80 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package report
import "github.com/goharbor/harbor/src/pkg/scan/dao/scan"
// Manager is used to manage the scan reports.
type Manager interface {
// Create a new report record.
//
// Arguments:
// r *scan.Report : report model to be created
//
// Returns:
// string : uuid of the new report
// error : non nil error if any errors occurred
//
Create(r *scan.Report) (string, error)
// Update the scan job ID of the given report.
//
// Arguments:
// uuid string : uuid to identify the report
// jobID string: scan job ID
//
// Returns:
// error : non nil error if any errors occurred
//
UpdateScanJobID(uuid string, jobID string) error
// Update the status (mapping to the scan job status) of the given report.
//
// Arguments:
// uuid string : uuid to identify the report
// status string: status info
// rev int64 : data revision info
//
// Returns:
// error : non nil error if any errors occurred
//
UpdateStatus(uuid string, status string, rev int64) error
// Update the report data (with JSON format) of the given report.
//
// Arguments:
// uuid string : uuid to identify the report
// report string: report JSON data
// rev int64 : data revision info
//
// Returns:
// error : non nil error if any errors occurred
//
UpdateReportData(uuid string, report string, rev int64) error
// Get the reports for the given digest by other properties.
//
// Arguments:
// digest string : digest of the artifact
// registrationUUID string : [optional] the report generated by which registration.
// If it is empty, reports by all the registrations are retrieved.
// mimeTypes []string : [optional] mime types of the reports requiring
// If empty array is specified, reports with all the supported mimes are retrieved.
//
// Returns:
// []*scan.Report : report list
// error : non nil error if any errors occurred
GetBy(digest string, registrationUUID string, mimeTypes []string) ([]*scan.Report, error)
}

4
src/pkg/scan/rest/auth/bearer_auth.go
View File

@ -17,6 +17,8 @@ package auth
import (
"fmt"
"net/http"
"github.com/pkg/errors"
)
// bearerAuthorizer authorizes the request by adding `Authorization Bearer credential` header
@ -31,7 +33,7 @@ func (ba *bearerAuthorizer) Authorize(req *http.Request) error {
req.Header.Add(authorization, fmt.Sprintf("%s %s", ba.typeID, ba.accessCred))
}
return nil
return errors.Errorf("%s: %s", ba.typeID, "missing data to authorize request")
}
// NewBearerAuth create bearer authorizer

131
src/pkg/scan/rest/v1/client_pool.go
View File

@ -15,16 +15,24 @@
package v1
import (
"encoding/base64"
"fmt"
"os"
"os/signal"
"sync"
"syscall"
"time"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/pkg/errors"
)
const (
defaultDeadCheckInterval = 1 * time.Minute
defaultExpireTime = 5 * time.Minute
)
// DefaultClientPool is a default client pool.
var DefaultClientPool = NewClientPool()
var DefaultClientPool = NewClientPool(nil)
// ClientPool defines operations for the client pool which provides v1 client cache.
type ClientPool interface {
@ -39,16 +47,47 @@ type ClientPool interface {
Get(r *scanner.Registration) (Client, error)
}
// PoolConfig provides configurations for the client pool.
type PoolConfig struct {
// Interval for checking dead instance.
DeadCheckInterval time.Duration
// Expire time for the instance to be marked as dead.
ExpireTime time.Duration
}
// poolItem append timestamp for the caching client instance.
type poolItem struct {
c Client
timestamp time.Time
}
// basicClientPool is default implementation of client pool interface.
type basicClientPool struct {
pool *sync.Map
pool *sync.Map
config *PoolConfig
}
// NewClientPool news a basic client pool.
func NewClientPool() ClientPool {
return &basicClientPool{
pool: &sync.Map{},
func NewClientPool(config *PoolConfig) ClientPool {
bcp := &basicClientPool{
pool: &sync.Map{},
config: config,
}
// Set config
if bcp.config == nil {
bcp.config = &PoolConfig{}
}
if bcp.config.DeadCheckInterval == 0 {
bcp.config.DeadCheckInterval = defaultDeadCheckInterval
}
if bcp.config.ExpireTime == 0 {
bcp.config.ExpireTime = defaultExpireTime
}
return bcp
}
// Get client for the specified registration.
@ -57,38 +96,7 @@ func NewClientPool() ClientPool {
// If one day, we have to clear unactivated client instances in the pool,
// add the following func after the first time initializing the client.
// pool item represents the client with a timestamp of last accessed.
//
// type poolItem struct {
// c Client
// timestamp time.Time
// }
//
// func (bcp *basicClientPool) deadCheck(key string, item *poolItem) {
// // Run in a separate goroutine
// go func() {
// // As we do not have a global context, let's watch the system signal to
// // exit the goroutine correctly.
// sig := make(chan os.Signal, 1)
// signal.Notify(sig, os.Interrupt, syscall.SIGTERM, os.Kill)
//
// tk := time.NewTicker(bcp.config.DeadCheckInterval)
// defer tk.Stop()
//
// for {
// select {
// case t := <-tk.C:
// if item.timestamp.Add(bcp.config.ExpireTime).Before(t.UTC()) {
// // Expired
// bcp.pool.Delete(key)
// return
// }
// case <-sig:
// // Terminated by system
// return
// }
// }
// }()
// }
func (bcp *basicClientPool) Get(r *scanner.Registration) (Client, error) {
if r == nil {
return nil, errors.New("nil scanner registration")
@ -100,7 +108,7 @@ func (bcp *basicClientPool) Get(r *scanner.Registration) (Client, error) {
k := key(r)
c, ok := bcp.pool.Load(k)
item, ok := bcp.pool.Load(k)
if !ok {
nc, err := NewClient(r)
if err != nil {
@ -108,21 +116,54 @@ func (bcp *basicClientPool) Get(r *scanner.Registration) (Client, error) {
}
// Cache it
bcp.pool.Store(k, nc)
c = nc
npi := &poolItem{
c: nc,
timestamp: time.Now().UTC(),
}
bcp.pool.Store(k, npi)
item = npi
// dead check
bcp.deadCheck(k, npi)
}
return c.(Client), nil
return item.(*poolItem).c, nil
}
func (bcp *basicClientPool) deadCheck(key string, item *poolItem) {
// Run in a separate goroutine
go func() {
// As we do not have a global context, let's watch the system signal to
// exit the goroutine correctly.
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt, syscall.SIGTERM, os.Kill)
tk := time.NewTicker(bcp.config.DeadCheckInterval)
defer tk.Stop()
for {
select {
case t := <-tk.C:
if item.timestamp.Add(bcp.config.ExpireTime).Before(t.UTC()) {
// Expired
bcp.pool.Delete(key)
return
}
case <-sig:
// Terminated by system
return
}
}
}()
}
func key(r *scanner.Registration) string {
raw := fmt.Sprintf("%s:%s:%s:%s:%v",
return fmt.Sprintf("%s:%s:%s:%s:%v",
r.UUID,
r.URL,
r.Auth,
r.AccessCredential,
r.SkipCertVerify,
)
return base64.StdEncoding.EncodeToString([]byte(raw))
}

17
src/pkg/scan/rest/v1/client_pool_test.go
View File

@ -17,6 +17,7 @@ package v1
import (
"fmt"
"testing"
"time"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/rest/auth"
@ -34,12 +35,16 @@ type ClientPoolTestSuite struct {
// TestClientPool is the entry of ClientPoolTestSuite.
func TestClientPool(t *testing.T) {
suite.Run(t, new(ClientPoolTestSuite))
suite.Run(t, &ClientPoolTestSuite{})
}
// SetupSuite sets up test suite env.
func (suite *ClientPoolTestSuite) SetupSuite() {
suite.pool = NewClientPool()
cfg := &PoolConfig{
DeadCheckInterval: 100 * time.Millisecond,
ExpireTime: 300 * time.Millisecond,
}
suite.pool = NewClientPool(cfg)
}
// TestClientPoolGet tests the get method of client pool.
@ -66,4 +71,12 @@ func (suite *ClientPoolTestSuite) TestClientPoolGet() {
p2 := fmt.Sprintf("%p", client2.(*basicClient))
assert.Equal(suite.T(), p1, p2)
<-time.After(400 * time.Millisecond)
client3, err := suite.pool.Get(r)
require.NoError(suite.T(), err)
require.NotNil(suite.T(), client3)
p3 := fmt.Sprintf("%p", client3.(*basicClient))
assert.NotEqual(suite.T(), p2, p3)
}

10
src/pkg/scan/rest/v1/spec.go
View File

@ -60,19 +60,17 @@ type Spec struct {
// NewSpec news V1 spec
func NewSpec(base string) *Spec {
baseRoute := "http://localhost"
s := &Spec{}
if len(base) > 0 {
if strings.HasSuffix(base, "/") {
baseRoute = base[:len(base)-1]
s.baseRoute = base[:len(base)-1]
} else {
baseRoute = base
s.baseRoute = base
}
}
return &Spec{
baseRoute: baseRoute,
}
return s
}
// Metadata API

2
src/pkg/scan/scanner/manager_test.go
View File

@ -63,7 +63,7 @@ func (suite *BasicManagerTestSuite) TearDownSuite() {
// TestList tests list registrations
func (suite *BasicManagerTestSuite) TestList() {
m := make(map[string]string, 1)
m := make(map[string]interface{}, 1)
m["name"] = "forUT"
l, err := suite.mgr.List(&q.Query{