From 0c19eba8c24eee993dc2fcc99476c3a566ddfb99 Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Thu, 19 Sep 2019 19:15:37 +0800 Subject: [PATCH] [stage2]support pluggable scanner - add scanner rest API v1 spec - implement v1 client which is used to talk to scanner adapter - adjust data/orm models - adjust code package structure Signed-off-by: Steven Zou - implement scan client which is used to talk to scanner adapter - implement scan job which take the work of communicating with scanner - update scanner mgmt API routes - add corresponding UT cases --- .../postgresql/0011_1.10.0_schema.up.sql | 12 +- src/core/api/harborapi_test.go | 2 +- .../api/{plug_scanners.go => scanners.go} | 14 +- ...plug_scanners_test.go => scanners_test.go} | 69 +--- src/core/router.go | 2 +- src/pkg/scan/api/scan/base_controller.go | 58 +++ src/pkg/scan/api/scan/controller.go | 78 ++++ .../scanner/base_controller.go} | 35 +- .../api => api/scanner}/controller.go | 46 +-- .../api => api/scanner}/controller_test.go | 7 +- .../dao/scan/report.go => dao/scan/model.go} | 14 +- .../scan/{scanner => }/dao/scanner/model.go | 17 +- .../{scanner => }/dao/scanner/model_test.go | 8 +- .../{scanner => }/dao/scanner/registration.go | 0 .../dao/scanner/registration_test.go | 7 +- src/pkg/scan/job.go | 337 ++++++++++++++++++ src/pkg/scan/job_test.go | 306 ++++++++++++++++ src/pkg/scan/report/supported_mime_test.go | 78 ++++ src/pkg/scan/report/supported_mimes.go | 56 +++ src/pkg/scan/rest/auth/api_key_auth.go | 45 +++ src/pkg/scan/rest/auth/auth.go | 54 +++ src/pkg/scan/rest/auth/basic_auth.go | 48 +++ src/pkg/scan/rest/auth/bearer_auth.go | 40 +++ .../api/scan.go => rest/auth/no_auth.go} | 24 +- src/pkg/scan/rest/v1/client.go | 278 +++++++++++++++ src/pkg/scan/rest/v1/client_pool.go | 128 +++++++ src/pkg/scan/rest/v1/client_pool_test.go | 69 ++++ src/pkg/scan/rest/v1/client_test.go | 197 ++++++++++ src/pkg/scan/rest/v1/models.go | 173 +++++++++ src/pkg/scan/rest/v1/spec.go | 109 ++++++ src/pkg/scan/scanner/manager.go | 8 +- src/pkg/scan/scanner/manager_test.go | 5 +- src/pkg/scan/scanner/scan/controller.go | 48 --- src/pkg/scan/scanner/scan/models.go | 46 --- src/pkg/scan/vuln/report.go | 58 +++ src/pkg/scan/vuln/severity.go | 39 ++ 36 files changed, 2221 insertions(+), 294 deletions(-) rename src/core/api/{plug_scanners.go => scanners.go} (96%) rename src/core/api/{plug_scanners_test.go => scanners_test.go} (85%) create mode 100644 src/pkg/scan/api/scan/base_controller.go create mode 100644 src/pkg/scan/api/scan/controller.go rename src/pkg/scan/{scanner/api/registration.go => api/scanner/base_controller.go} (86%) rename src/pkg/scan/{scanner/api => api/scanner}/controller.go (73%) rename src/pkg/scan/{scanner/api => api/scanner}/controller_test.go (98%) rename src/pkg/scan/{scanner/dao/scan/report.go => dao/scan/model.go} (76%) rename src/pkg/scan/{scanner => }/dao/scanner/model.go (86%) rename src/pkg/scan/{scanner => }/dao/scanner/model_test.go (92%) rename src/pkg/scan/{scanner => }/dao/scanner/registration.go (100%) rename src/pkg/scan/{scanner => }/dao/scanner/registration_test.go (96%) create mode 100644 src/pkg/scan/job.go create mode 100644 src/pkg/scan/job_test.go create mode 100644 src/pkg/scan/report/supported_mime_test.go create mode 100644 src/pkg/scan/report/supported_mimes.go create mode 100644 src/pkg/scan/rest/auth/api_key_auth.go create mode 100644 src/pkg/scan/rest/auth/auth.go create mode 100644 src/pkg/scan/rest/auth/basic_auth.go create mode 100644 src/pkg/scan/rest/auth/bearer_auth.go rename src/pkg/scan/{scanner/api/scan.go => rest/auth/no_auth.go} (58%) create mode 100644 src/pkg/scan/rest/v1/client.go create mode 100644 src/pkg/scan/rest/v1/client_pool.go create mode 100644 src/pkg/scan/rest/v1/client_pool_test.go create mode 100644 src/pkg/scan/rest/v1/client_test.go create mode 100644 src/pkg/scan/rest/v1/models.go create mode 100644 src/pkg/scan/rest/v1/spec.go delete mode 100644 src/pkg/scan/scanner/scan/controller.go delete mode 100644 src/pkg/scan/scanner/scan/models.go create mode 100644 src/pkg/scan/vuln/report.go create mode 100644 src/pkg/scan/vuln/severity.go diff --git a/make/migrations/postgresql/0011_1.10.0_schema.up.sql b/make/migrations/postgresql/0011_1.10.0_schema.up.sql index e03f84b09..ad4c5c35a 100644 --- a/make/migrations/postgresql/0011_1.10.0_schema.up.sql +++ b/make/migrations/postgresql/0011_1.10.0_schema.up.sql @@ -8,9 +8,6 @@ CREATE TABLE scanner_registration description VARCHAR(1024) NULL, auth VARCHAR(16) NOT NULL, access_cred VARCHAR(512) NULL, - adapter VARCHAR(128) NOT NULL, - vendor VARCHAR(128) NOT NULL, - version VARCHAR(32) NOT NULL, disabled BOOLEAN NOT NULL DEFAULT FALSE, is_default BOOLEAN NOT NULL DEFAULT FALSE, skip_cert_verify BOOLEAN NOT NULL DEFAULT FALSE, @@ -18,17 +15,18 @@ CREATE TABLE scanner_registration update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); -/*Table for keeping the scanner report. The report details are stored as JSONB*/ -CREATE TABLE scanner_report +/*Table for keeping the scan report. The report details are stored as JSON*/ +CREATE TABLE scan_report ( id SERIAL PRIMARY KEY NOT NULL, digest VARCHAR(256) NOT NULL, - registration_id VARCHAR(64) NOT NULL, + registration_uuid VARCHAR(64) NOT NULL, + mime_type VARCHAR(256) NOT NULL, job_id VARCHAR(32), status VARCHAR(16) NOT NULL, status_code INTEGER DEFAULT 0, report JSON, start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, end_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(digest, registration_id) + UNIQUE(digest, registration_uuid, mime_type) ) diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index b13fb2f30..be9a62050 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -209,7 +209,7 @@ func init() { // Add routes for plugin scanner management scannerAPI := &ScannerAPI{} beego.Router("/api/scanners", scannerAPI, "post:Create;get:List") - beego.Router("/api/scanners/:uid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault") + beego.Router("/api/scanners/:uuid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault") // Add routes for project level scanner beego.Router("/api/projects/:pid([0-9]+)/scanner", scannerAPI, "get:GetProjectScanner;put:SetProjectScanner") diff --git a/src/core/api/plug_scanners.go b/src/core/api/scanners.go similarity index 96% rename from src/core/api/plug_scanners.go rename to src/core/api/scanners.go index d14424675..34a2ecb89 100644 --- a/src/core/api/plug_scanners.go +++ b/src/core/api/scanners.go @@ -19,8 +19,8 @@ import ( "net/http" "github.com/goharbor/harbor/src/pkg/q" - "github.com/goharbor/harbor/src/pkg/scan/scanner/api" - "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner" + s "github.com/goharbor/harbor/src/pkg/scan/api/scanner" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" "github.com/pkg/errors" ) @@ -30,7 +30,7 @@ type ScannerAPI struct { BaseController // Controller for the plug scanners - c api.Controller + c s.Controller } // Prepare sth. for the subsequent actions @@ -50,7 +50,7 @@ func (sa *ScannerAPI) Prepare() { } // Use the default controller - sa.c = api.DefaultController + sa.c = s.DefaultController } // Get the specified scanner @@ -192,7 +192,7 @@ func (sa *ScannerAPI) Update() { // Delete the scanner func (sa *ScannerAPI) Delete() { - uid := sa.GetStringFromPath(":uid") + uid := sa.GetStringFromPath(":uuid") if len(uid) == 0 { sa.SendBadRequestError(errors.New("missing uid")) return @@ -216,7 +216,7 @@ func (sa *ScannerAPI) Delete() { // SetAsDefault sets the given registration as default one func (sa *ScannerAPI) SetAsDefault() { - uid := sa.GetStringFromPath(":uid") + uid := sa.GetStringFromPath(":uuid") if len(uid) == 0 { sa.SendBadRequestError(errors.New("missing uid")) return @@ -293,7 +293,7 @@ func (sa *ScannerAPI) SetProjectScanner() { // get the specified scanner func (sa *ScannerAPI) get() *scanner.Registration { - uid := sa.GetStringFromPath(":uid") + uid := sa.GetStringFromPath(":uuid") if len(uid) == 0 { sa.SendBadRequestError(errors.New("missing uid")) return nil diff --git a/src/core/api/plug_scanners_test.go b/src/core/api/scanners_test.go similarity index 85% rename from src/core/api/plug_scanners_test.go rename to src/core/api/scanners_test.go index f6beb9530..f0f7d3edd 100644 --- a/src/core/api/plug_scanners_test.go +++ b/src/core/api/scanners_test.go @@ -20,10 +20,8 @@ import ( "testing" "github.com/goharbor/harbor/src/pkg/q" - "github.com/goharbor/harbor/src/pkg/scan/scanner/api" - dscan "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scan" - "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner" - "github.com/goharbor/harbor/src/pkg/scan/scanner/scan" + sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -38,7 +36,7 @@ const ( type ScannerAPITestSuite struct { suite.Suite - originC api.Controller + originC sc.Controller mockC *MockScannerAPIController } @@ -49,9 +47,9 @@ func TestScannerAPI(t *testing.T) { // SetupSuite prepares testing env func (suite *ScannerAPITestSuite) SetupTest() { - suite.originC = api.DefaultController + suite.originC = sc.DefaultController m := &MockScannerAPIController{} - api.DefaultController = m + sc.DefaultController = m suite.mockC = m } @@ -59,7 +57,7 @@ func (suite *ScannerAPITestSuite) SetupTest() { // TearDownTest clears test case env func (suite *ScannerAPITestSuite) TearDownTest() { // Restore - api.DefaultController = suite.originC + sc.DefaultController = suite.originC } // TestScannerAPICreate tests the post request to create new one @@ -108,9 +106,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPIGet() { Name: "TestScannerAPIGet", Description: "JUST FOR TEST", URL: "https://a.b.c", - Adapter: "Clair", - Vendor: "Harbor", - Version: "0.1.0", } suite.mockC.On("GetRegistration", "uuid").Return(res, nil) @@ -133,9 +128,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPICreate() { Name: "TestScannerAPICreate", Description: "JUST FOR TEST", URL: "https://a.b.c", - Adapter: "Clair", - Vendor: "Harbor", - Version: "0.1.0", } suite.mockQuery(r) @@ -170,9 +162,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPIList() { Name: "TestScannerAPIList", Description: "JUST FOR TEST", URL: "https://a.b.c", - Adapter: "Clair", - Vendor: "Harbor", - Version: "0.1.0", }} suite.mockC.On("ListRegistrations", query).Return(ll, nil) @@ -198,9 +187,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPIUpdate() { Name: "TestScannerAPIUpdate_before", Description: "JUST FOR TEST", URL: "https://a.b.c", - Adapter: "Clair", - Vendor: "Harbor", - Version: "0.1.0", } updated := &scanner.Registration{ @@ -209,9 +195,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPIUpdate() { Name: "TestScannerAPIUpdate", Description: "JUST FOR TEST", URL: "https://a.b.c", - Adapter: "Clair", - Vendor: "Harbor", - Version: "0.1.0", } suite.mockQuery(updated) @@ -240,9 +223,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPIDelete() { Name: "TestScannerAPIDelete", Description: "JUST FOR TEST", URL: "https://a.b.c", - Adapter: "Clair", - Vendor: "Harbor", - Version: "0.1.0", } suite.mockC.On("DeleteRegistration", "uuid").Return(r, nil) @@ -299,9 +279,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPIProjectScanner() { Name: "TestScannerAPIProjectScanner", Description: "JUST FOR TEST", URL: "https://a.b.c", - Adapter: "Clair", - Vendor: "Harbor", - Version: "0.1.0", } suite.mockC.On("GetRegistrationByProject", int64(1)).Return(r, nil) @@ -408,37 +385,3 @@ func (m *MockScannerAPIController) GetRegistrationByProject(projectID int64) (*s return s.(*scanner.Registration), args.Error(1) } - -// Ping ... -func (m *MockScannerAPIController) Ping(registration *scanner.Registration) error { - args := m.Called(registration) - return args.Error(0) -} - -// Scan ... -func (m *MockScannerAPIController) Scan(artifact *scan.Artifact) error { - args := m.Called(artifact) - return args.Error(0) -} - -// GetReport ... -func (m *MockScannerAPIController) GetReport(artifact *scan.Artifact) ([]*dscan.Report, error) { - args := m.Called(artifact) - r := args.Get(0) - if r == nil { - return nil, args.Error(1) - } - - return r.([]*dscan.Report), args.Error(1) -} - -// GetScanLog ... -func (m *MockScannerAPIController) GetScanLog(digest string) ([]byte, error) { - args := m.Called(digest) - l := args.Get(0) - if l == nil { - return nil, args.Error(1) - } - - return l.([]byte), args.Error(1) -} diff --git a/src/core/router.go b/src/core/router.go index deb862c6a..7afe5f8f8 100755 --- a/src/core/router.go +++ b/src/core/router.go @@ -195,7 +195,7 @@ func initRouters() { // Add routes for plugin scanner management scannerAPI := &api.ScannerAPI{} beego.Router("/api/scanners", scannerAPI, "post:Create;get:List") - beego.Router("/api/scanners/:uid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault") + beego.Router("/api/scanners/:uuid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault") // Add routes for project level scanner beego.Router("/api/projects/:pid([0-9]+)/scanner", scannerAPI, "get:GetProjectScanner;put:SetProjectScanner") diff --git a/src/pkg/scan/api/scan/base_controller.go b/src/pkg/scan/api/scan/base_controller.go new file mode 100644 index 000000000..b3d6f8505 --- /dev/null +++ b/src/pkg/scan/api/scan/base_controller.go @@ -0,0 +1,58 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/pkg/scan/dao/scan" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" +) + +// basicController is default implementation of api.Controller interface +type basicController struct { + // Client for talking to scanner adapter + client v1.Client +} + +// NewController news a scan API controller +func NewController() Controller { + return &basicController{} +} + +// Scan ... +func (bc *basicController) Scan(artifact *v1.Artifact) error { + return nil +} + +// GetReport ... +func (bc *basicController) GetReport(artifact *v1.Artifact) ([]*scan.Report, error) { + return nil, nil +} + +// GetScanLog ... +func (bc *basicController) GetScanLog(digest string) ([]byte, error) { + return nil, nil +} + +// Ping ... +func (bc *basicController) Ping(registration *scanner.Registration) error { + return nil +} + +// HandleJobHooks ... +func (bc *basicController) HandleJobHooks(trackID int64, change *job.StatusChange) error { + return nil +} diff --git a/src/pkg/scan/api/scan/controller.go b/src/pkg/scan/api/scan/controller.go new file mode 100644 index 000000000..65322148b --- /dev/null +++ b/src/pkg/scan/api/scan/controller.go @@ -0,0 +1,78 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/pkg/scan/dao/scan" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" +) + +// Controller provides the related operations for triggering scan. +// TODO: Here the artifact object is reused the v1 one which is sent to the adapter, +// it should be pointed to the general artifact object in future once it's ready. +type Controller interface { + // Ping pings Scanner Adapter to test EndpointURL and Authorization settings. + // The implementation is supposed to call the GetMetadata method on scanner.Client. + // Returns `nil` if connection succeeded, a non `nil` error otherwise. + // + // Arguments: + // registration *scanner.Registration : scanner registration to ping + // + // Returns: + // error : non nil error if any errors occurred + Ping(registration *scanner.Registration) error + + // Scan the given artifact + // + // Arguments: + // artifact *v1.Artifact : artifact to be scanned + // + // Returns: + // error : non nil error if any errors occurred + Scan(artifact *v1.Artifact) error + + // GetReport gets the reports for the given artifact identified by the digest + // + // Arguments: + // artifact *v1.Artifact : the scanned artifact + // + // Returns: + // []*scan.Report : scan results by different scanner vendors + // error : non nil error if any errors occurred + GetReport(artifact *v1.Artifact) ([]*scan.Report, error) + + // Get the scan log for the specified artifact with the given digest + // + // Arguments: + // digest string : the digest of the artifact + // + // Returns: + // []byte : the log text stream + // error : non nil error if any errors occurred + GetScanLog(digest string) ([]byte, error) + + // HandleJobHooks handle the hook events from the job service + // e.g : status change of the scan job or scan result + // + // Arguments: + // trackID int64 : ID for the result record + // change *job.StatusChange : change event from the job service + // + // Returns: + // error : non nil error if any errors occurred + HandleJobHooks(trackID int64, change *job.StatusChange) error +} diff --git a/src/pkg/scan/scanner/api/registration.go b/src/pkg/scan/api/scanner/base_controller.go similarity index 86% rename from src/pkg/scan/scanner/api/registration.go rename to src/pkg/scan/api/scanner/base_controller.go index 879498915..72512ceff 100644 --- a/src/pkg/scan/scanner/api/registration.go +++ b/src/pkg/scan/api/scanner/base_controller.go @@ -12,15 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package api +package scanner import ( "github.com/goharbor/harbor/src/core/promgr/metamgr" "github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/pkg/q" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" rscanner "github.com/goharbor/harbor/src/pkg/scan/scanner" - "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner" - "github.com/goharbor/harbor/src/pkg/scan/scanner/scan" "github.com/pkg/errors" ) @@ -45,9 +44,6 @@ type basicController struct { manager rscanner.Manager // for operating the project level configured scanner proMetaMgr metamgr.ProjectMetadataManager - // controller for scan actions - c scan.Controller - // Client } // ListRegistrations ... @@ -58,9 +54,13 @@ func (bc *basicController) ListRegistrations(query *q.Query) ([]*scanner.Registr // CreateRegistration ... func (bc *basicController) CreateRegistration(registration *scanner.Registration) (string, error) { // TODO: Get metadata from the adapter service first - l, err := bc.manager.List(nil) + // Check if there are any registrations already existing. + l, err := bc.manager.List(&q.Query{ + PageSize: 1, + PageNumber: 1, + }) if err != nil { - return "", err + return "", errors.Wrap(err, "api controller: create registration") } if len(l) == 0 && !registration.IsDefault { @@ -102,7 +102,7 @@ func (bc *basicController) DeleteRegistration(registrationUUID string) (*scanner } if err := bc.manager.Delete(registrationUUID); err != nil { - return nil, errors.Wrap(err, "delete registration") + return nil, errors.Wrap(err, "api controller: delete registration") } return registration, nil @@ -127,7 +127,7 @@ func (bc *basicController) SetRegistrationByProject(projectID int64, registratio // Scanner metadata existing? m, err := bc.proMetaMgr.Get(projectID, proScannerMetaKey) if err != nil { - return errors.Wrap(err, "set project scanner") + return errors.Wrap(err, "api controller: set project scanner") } // Update if exists @@ -136,14 +136,14 @@ func (bc *basicController) SetRegistrationByProject(projectID int64, registratio if registrationID != m[proScannerMetaKey] { m[proScannerMetaKey] = registrationID if err := bc.proMetaMgr.Update(projectID, m); err != nil { - return errors.Wrap(err, "set project scanner") + return errors.Wrap(err, "api controller: set project scanner") } } } else { meta := make(map[string]string, 1) meta[proScannerMetaKey] = registrationID if err := bc.proMetaMgr.Add(projectID, meta); err != nil { - return errors.Wrap(err, "set project scanner") + return errors.Wrap(err, "api controller: set project scanner") } } @@ -159,21 +159,21 @@ func (bc *basicController) GetRegistrationByProject(projectID int64) (*scanner.R // First, get it from the project metadata m, err := bc.proMetaMgr.Get(projectID, proScannerMetaKey) if err != nil { - return nil, errors.Wrap(err, "get project scanner") + return nil, errors.Wrap(err, "api controller: get project scanner") } if len(m) > 0 { if registrationID, ok := m[proScannerMetaKey]; ok && len(registrationID) > 0 { registration, err := bc.manager.Get(registrationID) if err != nil { - return nil, errors.Wrap(err, "get project scanner") + return nil, errors.Wrap(err, "api controller: get project scanner") } if registration == nil { // Not found // Might be deleted by the admin, the project scanner ID reference should be cleared if err := bc.proMetaMgr.Delete(projectID, proScannerMetaKey); err != nil { - return nil, errors.Wrap(err, "get project scanner") + return nil, errors.Wrap(err, "api controller: get project scanner") } } else { return registration, nil @@ -187,8 +187,3 @@ func (bc *basicController) GetRegistrationByProject(projectID int64) (*scanner.R // TODO: Check status by the client later return registration, err } - -// Ping ... -func (bc *basicController) Ping(registration *scanner.Registration) error { - return nil -} diff --git a/src/pkg/scan/scanner/api/controller.go b/src/pkg/scan/api/scanner/controller.go similarity index 73% rename from src/pkg/scan/scanner/api/controller.go rename to src/pkg/scan/api/scanner/controller.go index 090dcde5f..048d94e96 100644 --- a/src/pkg/scan/scanner/api/controller.go +++ b/src/pkg/scan/api/scanner/controller.go @@ -12,13 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package api +package scanner import ( "github.com/goharbor/harbor/src/pkg/q" - dscan "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scan" - "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner" - "github.com/goharbor/harbor/src/pkg/scan/scanner/scan" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" ) // Controller provides the related operations of scanner for the upper API. @@ -114,44 +112,4 @@ type Controller interface { // *scanner.Registration : the default scanner registration // error : non nil error if any errors occurred GetRegistrationByProject(projectID int64) (*scanner.Registration, error) - - // Ping pings Scanner Adapter to test EndpointURL and Authorization settings. - // The implementation is supposed to call the GetMetadata method on scanner.Client. - // Returns `nil` if connection succeeded, a non `nil` error otherwise. - // - // Arguments: - // registration *scanner.Registration : scanner registration to ping - // - // Returns: - // error : non nil error if any errors occurred - Ping(registration *scanner.Registration) error - - // Scan the given artifact - // - // Arguments: - // artifact *res.Artifact : artifact to be scanned - // - // Returns: - // error : non nil error if any errors occurred - Scan(artifact *scan.Artifact) error - - // GetReport gets the reports for the given artifact identified by the digest - // - // Arguments: - // artifact *res.Artifact : the scanned artifact - // - // Returns: - // []*scan.Report : scan results by different scanner vendors - // error : non nil error if any errors occurred - GetReport(artifact *scan.Artifact) ([]*dscan.Report, error) - - // Get the scan log for the specified artifact with the given digest - // - // Arguments: - // digest string : the digest of the artifact - // - // Returns: - // []byte : the log text stream - // error : non nil error if any errors occurred - GetScanLog(digest string) ([]byte, error) } diff --git a/src/pkg/scan/scanner/api/controller_test.go b/src/pkg/scan/api/scanner/controller_test.go similarity index 98% rename from src/pkg/scan/scanner/api/controller_test.go rename to src/pkg/scan/api/scanner/controller_test.go index 0ea4c5ad2..e032b645d 100644 --- a/src/pkg/scan/scanner/api/controller_test.go +++ b/src/pkg/scan/api/scanner/controller_test.go @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package api +package scanner import ( "testing" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/pkg/q" - "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -56,9 +56,6 @@ func (suite *ControllerTestSuite) SetupSuite() { Name: "forUT", Description: "sample registration", URL: "https://sample.scanner.com", - Adapter: "Clair", - Version: "0.1.0", - Vendor: "Harbor", } } diff --git a/src/pkg/scan/scanner/dao/scan/report.go b/src/pkg/scan/dao/scan/model.go similarity index 76% rename from src/pkg/scan/scanner/dao/scan/report.go rename to src/pkg/scan/dao/scan/model.go index 13d3f6cc9..5789e632c 100644 --- a/src/pkg/scan/scanner/dao/scan/report.go +++ b/src/pkg/scan/dao/scan/model.go @@ -16,15 +16,18 @@ package scan import "time" -// Report of the scan -// Identified by the `digest` and `endpoint_id` +// Report of the scan. +// Identified by the `digest`, `registration_uuid` and `mime_type`. type Report struct { ID int64 `orm:"pk;auto;column(id)"` + UUID string `orm:"unique;column(uuid)"` Digest string `orm:"column(digest)"` - ReregistrationID string `orm:"column(registration_id)"` + RegistrationUUID string `orm:"column(registration_uuid)"` + MimeType string `orm:"column(mime_type)"` JobID string `orm:"column(job_id)"` Status string `orm:"column(status)"` StatusCode int `orm:"column(status_code)"` + StatusRevision int64 `orm:"column(status_rev)"` Report string `orm:"column(report);type(json)"` StartTime time.Time `orm:"column(start_time);auto_now_add;type(datetime)"` EndTime time.Time `orm:"column(end_time);type(datetime)"` @@ -32,12 +35,13 @@ type Report struct { // TableName for Report func (r *Report) TableName() string { - return "scanner_report" + return "scan_report" } // TableUnique for Report func (r *Report) TableUnique() [][]string { return [][]string{ - {"digest", "registration_id"}, + {"uuid"}, + {"digest", "registration_uuid", "mime_type"}, } } diff --git a/src/pkg/scan/scanner/dao/scanner/model.go b/src/pkg/scan/dao/scanner/model.go similarity index 86% rename from src/pkg/scan/scanner/dao/scanner/model.go rename to src/pkg/scan/dao/scanner/model.go index fa28b87df..bf711a7f3 100644 --- a/src/pkg/scan/scanner/dao/scanner/model.go +++ b/src/pkg/scan/dao/scanner/model.go @@ -45,11 +45,6 @@ type Registration struct { // Http connection settings SkipCertVerify bool `orm:"column(skip_cert_verify);default(false)" json:"skip_certVerify"` - // Adapter settings - Adapter string `orm:"column(adapter);size(128)" json:"adapter"` - Vendor string `orm:"column(vendor);size(128)" json:"vendor"` - Version string `orm:"column(version);size(32)" json:"version"` - // Timestamps CreateTime time.Time `orm:"column(create_time);auto_now_add;type(datetime)" json:"create_time"` UpdateTime time.Time `orm:"column(update_time);auto_now;type(datetime)" json:"update_time"` @@ -60,7 +55,7 @@ func (r *Registration) TableName() string { return "scanner_registration" } -// FromJSON parses json data +// FromJSON parses registration from json data func (r *Registration) FromJSON(jsonData string) error { if len(jsonData) == 0 { return errors.New("empty json data to parse") @@ -69,7 +64,7 @@ func (r *Registration) FromJSON(jsonData string) error { return json.Unmarshal([]byte(jsonData), r) } -// ToJSON marshals endpoint to JSON data +// ToJSON marshals registration to JSON data func (r *Registration) ToJSON() (string, error) { data, err := json.Marshal(r) if err != nil { @@ -79,7 +74,7 @@ func (r *Registration) ToJSON() (string, error) { return string(data), nil } -// Validate endpoint +// Validate registration func (r *Registration) Validate(checkUUID bool) error { if checkUUID && len(r.UUID) == 0 { return errors.New("malformed endpoint") @@ -94,12 +89,6 @@ func (r *Registration) Validate(checkUUID bool) error { return errors.Wrap(err, "scanner registration validate") } - if len(r.Adapter) == 0 || - len(r.Vendor) == 0 || - len(r.Version) == 0 { - return errors.Errorf("missing adapter settings in registration %s:%s", r.Name, r.URL) - } - return nil } diff --git a/src/pkg/scan/scanner/dao/scanner/model_test.go b/src/pkg/scan/dao/scanner/model_test.go similarity index 92% rename from src/pkg/scan/scanner/dao/scanner/model_test.go rename to src/pkg/scan/dao/scanner/model_test.go index 25d837f92..aaf185fff 100644 --- a/src/pkg/scan/scanner/dao/scanner/model_test.go +++ b/src/pkg/scan/dao/scanner/model_test.go @@ -38,9 +38,6 @@ func (suite *ModelTestSuite) TestJSON() { Name: "forUT", Description: "sample registration", URL: "https://sample.scanner.com", - Adapter: "Clair", - Version: "0.1.0", - Vendor: "Harbor", } json, err := r.ToJSON() @@ -77,11 +74,8 @@ func (suite *ModelTestSuite) TestValidate() { r.URL = "http://a.b.c" err = r.Validate(true) - require.Error(suite.T(), err) + require.NoError(suite.T(), err) - r.Adapter = "Clair" - r.Vendor = "Harbor" - r.Version = "0.1.0" err = r.Validate(true) require.NoError(suite.T(), err) } diff --git a/src/pkg/scan/scanner/dao/scanner/registration.go b/src/pkg/scan/dao/scanner/registration.go similarity index 100% rename from src/pkg/scan/scanner/dao/scanner/registration.go rename to src/pkg/scan/dao/scanner/registration.go diff --git a/src/pkg/scan/scanner/dao/scanner/registration_test.go b/src/pkg/scan/dao/scanner/registration_test.go similarity index 96% rename from src/pkg/scan/scanner/dao/scanner/registration_test.go rename to src/pkg/scan/dao/scanner/registration_test.go index 1575bf67f..c76a2efd2 100644 --- a/src/pkg/scan/scanner/dao/scanner/registration_test.go +++ b/src/pkg/scan/dao/scanner/registration_test.go @@ -50,9 +50,6 @@ func (suite *RegistrationDAOTestSuite) SetupTest() { Name: "forUT", Description: "sample registration", URL: "https://sample.scanner.com", - Adapter: "Clair", - Version: "0.1.0", - Vendor: "Harbor", } _, err := AddRegistration(r) @@ -111,7 +108,7 @@ func (suite *RegistrationDAOTestSuite) TestList() { // with query and found items keywords := make(map[string]string) - keywords["adapter"] = "Clair" + keywords["description"] = "sample" l, err = ListRegistrations(&q.Query{ PageSize: 5, PageNumber: 1, @@ -121,7 +118,7 @@ func (suite *RegistrationDAOTestSuite) TestList() { require.Equal(suite.T(), 1, len(l)) // With query and not found items - keywords["adapter"] = "Micro scanner" + keywords["description"] = "not_exist" l, err = ListRegistrations(&q.Query{ Keywords: keywords, }) diff --git a/src/pkg/scan/job.go b/src/pkg/scan/job.go new file mode 100644 index 000000000..f38dc53e5 --- /dev/null +++ b/src/pkg/scan/job.go @@ -0,0 +1,337 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "sync" + "time" + + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scan/report" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/pkg/errors" +) + +const ( + // JobParamRegistration ... + JobParamRegistration = "registration" + // JobParameterRequest ... + JobParameterRequest = "scanRequest" + // JobParameterMimes ... + JobParameterMimes = "mimeTypes" + + checkTimeout = 30 * time.Minute + firstCheckInterval = 2 * time.Second +) + +// CheckInReport defines model for checking in the scan report with specified mime. +type CheckInReport struct { + Digest string `json:"digest"` + RegistrationUUID string `json:"registration_uuid"` + MimeType string `json:"mime_type"` + RawReport string `json:"raw_report"` +} + +// FromJSON parse json to CheckInReport +func (cir *CheckInReport) FromJSON(jsonData string) error { + if len(jsonData) == 0 { + return errors.New("empty JSON data") + } + + return json.Unmarshal([]byte(jsonData), cir) +} + +// ToJSON marshal CheckInReport to JSON +func (cir *CheckInReport) ToJSON() (string, error) { + jsonData, err := json.Marshal(cir) + if err != nil { + return "", errors.Wrap(err, "To JSON: CheckInReport") + } + + return string(jsonData), nil +} + +// Job for running scan in the job service with async way +type Job struct{} + +// MaxFails for defining the number of retries +func (j *Job) MaxFails() uint { + return 3 +} + +// ShouldRetry indicates if the job should be retried +func (j *Job) ShouldRetry() bool { + return true +} + +// Validate the parameters of this job +func (j *Job) Validate(params job.Parameters) error { + if params == nil { + // Params are required + return errors.New("missing parameter of scan job") + } + + if _, err := extractRegistration(params); err != nil { + return errors.Wrap(err, "job validate") + } + + if _, err := extractScanReq(params); err != nil { + return errors.Wrap(err, "job validate") + } + + if _, err := extractMimeTypes(params); err != nil { + return errors.Wrap(err, "job validate") + } + + return nil +} + +// Run the job +func (j *Job) Run(ctx job.Context, params job.Parameters) error { + // Get logger + myLogger := ctx.GetLogger() + + // Ignore errors as they have been validated already + r, _ := extractRegistration(params) + req, _ := extractScanReq(params) + mimes, _ := extractMimeTypes(params) + + // Print related infos to log + printJSONParameter(JobParamRegistration, params[JobParamRegistration].(string), myLogger) + printJSONParameter(JobParameterRequest, params[JobParameterRequest].(string), myLogger) + + // Submit scan request to the scanner adapter + client, err := v1.DefaultClientPool.Get(r) + if err != nil { + return errors.Wrap(err, "run scan job") + } + + resp, err := client.SubmitScan(req) + if err != nil { + return errors.Wrap(err, "run scan job") + } + + // Exit signal + exit := make(chan bool, 1) + // Done signal + done := make(chan bool, 1) + // Collect errors + eChan := make(chan error, 1) + + go func() { + defer func() { + // Done! + done <- true + }() + + for { + select { + case e := <-eChan: + if err != nil { + err = errors.Wrap(e, err.Error()) + } else { + err = e + } + case <-exit: + // Gracefully exit + return + case <-ctx.SystemContext().Done(): + // Terminated by system + return + } + } + }() + + // Concurrently retrieving report by different mime types + wg := &sync.WaitGroup{} + wg.Add(len(mimes)) + + for _, mt := range mimes { + go func(m string) { + defer wg.Done() + + // Log info + myLogger.Infof("Get report for mime type: %s", m) + + // Loop check if the report is ready + tm := time.NewTimer(firstCheckInterval) + defer tm.Stop() + + for { + select { + case t := <-tm.C: + myLogger.Debugf("check scan report for mime %s at %s", m, t.Format("2006/01/02 15:04:05")) + + rawReport, err := client.GetScanReport(resp.ID, m) + if err != nil { + // Not ready yet + if notReadyErr, ok := err.(*v1.ReportNotReadyError); ok { + // Reset to the new check interval + tm.Reset(time.Duration(notReadyErr.RetryAfter) * time.Second) + myLogger.Infof("Report with mime type %s is not ready yet, retry after %d seconds", m, notReadyErr.RetryAfter) + + continue + } + + eChan <- errors.Wrap(err, fmt.Sprintf("check scan report with mime type %s", m)) + return + } + + // Make sure the data is aligned with the v1 spec. + if _, err = report.ResolveData(m, []byte(rawReport)); err != nil { + eChan <- errors.Wrap(err, "scan job: resolve report data") + return + } + + // Check in + cir := &CheckInReport{ + Digest: req.Artifact.Digest, + RegistrationUUID: r.UUID, + MimeType: m, + RawReport: rawReport, + } + + var ( + jsonData string + er error + ) + if jsonData, er = cir.ToJSON(); er == nil { + if er = ctx.Checkin(jsonData); er == nil { + // Done! + myLogger.Infof("Report with mime type %s is checked in", m) + return + } + } + + // Send error and exit + eChan <- errors.Wrap(er, fmt.Sprintf("check in scan report for mime type %s", m)) + return + case <-ctx.SystemContext().Done(): + // Terminated by system + return + case <-time.After(checkTimeout): + eChan <- errors.New("check scan report timeout") + return + } + } + }(mt) + } + + // Wait for all the retrieving routines are completed + wg.Wait() + // Stop error collection goroutine + exit <- true + // done! + <-done + // Log error to the job log + if err != nil { + myLogger.Error(err) + } + + return err +} + +func printJSONParameter(parameter string, v string, logger logger.Interface) { + logger.Infof("%s:\n", parameter) + printPrettyJSON([]byte(v), logger) +} + +func printPrettyJSON(in []byte, logger logger.Interface) { + var out bytes.Buffer + if err := json.Indent(&out, in, "", " "); err != nil { + logger.Errorf("Print pretty JSON error: %s", err) + return + } + + logger.Infof("%s\n", out.String()) +} + +func extractScanReq(params job.Parameters) (*v1.ScanRequest, error) { + v, ok := params[JobParameterRequest] + if !ok { + return nil, errors.Errorf("missing job parameter '%s'", JobParameterRequest) + } + + jsonData, ok := v.(string) + if !ok { + return nil, errors.Errorf( + "malformed job parameter '%s', expecting string but got %s", + JobParameterRequest, + reflect.TypeOf(v).String(), + ) + } + + req := &v1.ScanRequest{} + if err := req.FromJSON(jsonData); err != nil { + return nil, err + } + + if err := req.Validate(); err != nil { + return nil, err + } + + return req, nil +} + +func extractRegistration(params job.Parameters) (*scanner.Registration, error) { + v, ok := params[JobParamRegistration] + if !ok { + return nil, errors.Errorf("missing job parameter '%s'", JobParamRegistration) + } + + jsonData, ok := v.(string) + if !ok { + return nil, errors.Errorf( + "malformed job parameter '%s', expecting string but got %s", + JobParamRegistration, + reflect.TypeOf(v).String(), + ) + } + + r := &scanner.Registration{} + if err := r.FromJSON(jsonData); err != nil { + return nil, err + } + + if err := r.Validate(true); err != nil { + return nil, err + } + + return r, nil +} + +func extractMimeTypes(params job.Parameters) ([]string, error) { + v, ok := params[JobParameterMimes] + if !ok { + return nil, errors.Errorf("missing job parameter '%s'", JobParameterMimes) + } + + l, ok := v.([]string) + if !ok { + return nil, errors.Errorf( + "malformed job parameter '%s', expecting string but got %s", + JobParameterMimes, + reflect.TypeOf(v).String(), + ) + } + + return l, nil +} diff --git a/src/pkg/scan/job_test.go b/src/pkg/scan/job_test.go new file mode 100644 index 000000000..26af182b3 --- /dev/null +++ b/src/pkg/scan/job_test.go @@ -0,0 +1,306 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/goharbor/harbor/src/pkg/scan/vuln" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// JobTestSuite is a test suite to test the scan job. +type JobTestSuite struct { + suite.Suite + + defaultClientPool v1.ClientPool + mcp *MockClientPool +} + +// TestJob is the entry of JobTestSuite. +func TestJob(t *testing.T) { + suite.Run(t, &JobTestSuite{}) +} + +// SetupSuite sets up test env for JobTestSuite. +func (suite *JobTestSuite) SetupSuite() { + mcp := &MockClientPool{} + suite.defaultClientPool = v1.DefaultClientPool + v1.DefaultClientPool = mcp + + suite.mcp = mcp +} + +// TeraDownSuite clears test env for TeraDownSuite. +func (suite *JobTestSuite) TeraDownSuite() { + v1.DefaultClientPool = suite.defaultClientPool +} + +// TestJob tests the scan job +func (suite *JobTestSuite) TestJob() { + ctx := &MockJobContext{} + lg := &MockJobLogger{} + + ctx.On("GetLogger").Return(lg) + + r := &scanner.Registration{ + ID: 0, + UUID: "uuid", + Name: "TestJob", + URL: "https://clair.com:8080", + } + + rData, err := r.ToJSON() + require.NoError(suite.T(), err) + + sr := &v1.ScanRequest{ + Registry: &v1.Registry{ + URL: "http://localhost:5000", + Authorization: "the_token", + }, + Artifact: &v1.Artifact{ + Repository: "library/test_job", + Digest: "sha256:data", + MimeType: v1.MimeTypeDockerArtifact, + }, + } + + sData, err := sr.ToJSON() + require.NoError(suite.T(), err) + + mimeTypes := []string{v1.MimeTypeNativeReport} + + jp := make(job.Parameters) + jp[JobParamRegistration] = rData + jp[JobParameterRequest] = sData + jp[JobParameterMimes] = mimeTypes + + mc := &MockClient{} + sre := &v1.ScanResponse{ + ID: "scan_id", + } + mc.On("SubmitScan", sr).Return(sre, nil) + + rp := vuln.Report{ + GeneratedAt: time.Now().UTC().String(), + Scanner: &v1.Scanner{ + Name: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + }, + Severity: vuln.High, + Vulnerabilities: []*vuln.VulnerabilityItem{ + { + ID: "2019-0980-0909", + Package: "dpkg", + Version: "0.9.1", + FixVersion: "0.9.2", + Severity: vuln.High, + Description: "mock one", + Links: []string{"https://vuln.com"}, + }, + }, + } + + jRep, err := json.Marshal(rp) + require.NoError(suite.T(), err) + + mc.On("GetScanReport", "scan_id", v1.MimeTypeNativeReport).Return(string(jRep), nil) + suite.mcp.On("Get", r).Return(mc, nil) + + crp := &CheckInReport{ + Digest: sr.Artifact.Digest, + RegistrationUUID: r.UUID, + MimeType: v1.MimeTypeNativeReport, + RawReport: string(jRep), + } + + jsonData, err := crp.ToJSON() + require.NoError(suite.T(), err) + + ctx.On("Checkin", string(jsonData)).Return(nil) + j := &Job{} + err = j.Run(ctx, jp) + require.NoError(suite.T(), err) +} + +// MockJobContext mocks job context interface. +// TODO: Maybe moved to a separate `mock` pkg for sharing in future. +type MockJobContext struct { + mock.Mock +} + +// Build ... +func (mjc *MockJobContext) Build(tracker job.Tracker) (job.Context, error) { + args := mjc.Called(tracker) + c := args.Get(0) + if c != nil { + return c.(job.Context), nil + } + + return nil, args.Error(1) +} + +// Get ... +func (mjc *MockJobContext) Get(prop string) (interface{}, bool) { + args := mjc.Called(prop) + return args.Get(0), args.Bool(1) +} + +// SystemContext ... +func (mjc *MockJobContext) SystemContext() context.Context { + return context.TODO() +} + +// Checkin ... +func (mjc *MockJobContext) Checkin(status string) error { + args := mjc.Called(status) + return args.Error(0) +} + +// OPCommand ... +func (mjc *MockJobContext) OPCommand() (job.OPCommand, bool) { + args := mjc.Called() + return (job.OPCommand)(args.String(0)), args.Bool(1) +} + +// GetLogger ... +func (mjc *MockJobContext) GetLogger() logger.Interface { + return &MockJobLogger{} +} + +// Tracker ... +func (mjc *MockJobContext) Tracker() job.Tracker { + args := mjc.Called() + if t := args.Get(0); t != nil { + return t.(job.Tracker) + } + + return nil +} + +// MockJobLogger mocks the job logger interface. +// TODO: Maybe moved to a separate `mock` pkg for sharing in future. +type MockJobLogger struct { + mock.Mock +} + +// Debug ... +func (mjl *MockJobLogger) Debug(v ...interface{}) { + logger.Debug(v...) +} + +// Debugf ... +func (mjl *MockJobLogger) Debugf(format string, v ...interface{}) { + logger.Debugf(format, v...) +} + +// Info ... +func (mjl *MockJobLogger) Info(v ...interface{}) { + logger.Info(v...) +} + +// Infof ... +func (mjl *MockJobLogger) Infof(format string, v ...interface{}) { + logger.Infof(format, v...) +} + +// Warning ... +func (mjl *MockJobLogger) Warning(v ...interface{}) { + logger.Warning(v...) +} + +// Warningf ... +func (mjl *MockJobLogger) Warningf(format string, v ...interface{}) { + logger.Warningf(format, v...) +} + +// Error ... +func (mjl *MockJobLogger) Error(v ...interface{}) { + logger.Error(v...) +} + +// Errorf ... +func (mjl *MockJobLogger) Errorf(format string, v ...interface{}) { + logger.Errorf(format, v...) +} + +// Fatal ... +func (mjl *MockJobLogger) Fatal(v ...interface{}) { + logger.Fatal(v...) +} + +// Fatalf ... +func (mjl *MockJobLogger) Fatalf(format string, v ...interface{}) { + logger.Fatalf(format, v...) +} + +// MockClientPool mocks the client pool +type MockClientPool struct { + mock.Mock +} + +// Get v1 client +func (mcp *MockClientPool) Get(r *scanner.Registration) (v1.Client, error) { + args := mcp.Called(r) + c := args.Get(0) + if c != nil { + return c.(v1.Client), nil + } + + return nil, args.Error(1) +} + +// MockClient mocks the v1 client +type MockClient struct { + mock.Mock +} + +// GetMetadata ... +func (mc *MockClient) GetMetadata() (*v1.ScannerAdapterMetadata, error) { + args := mc.Called() + s := args.Get(0) + if s != nil { + return s.(*v1.ScannerAdapterMetadata), nil + } + + return nil, args.Error(1) +} + +// SubmitScan ... +func (mc *MockClient) SubmitScan(req *v1.ScanRequest) (*v1.ScanResponse, error) { + args := mc.Called(req) + sr := args.Get(0) + if sr != nil { + return sr.(*v1.ScanResponse), nil + } + + return nil, args.Error(1) +} + +// GetScanReport ... +func (mc *MockClient) GetScanReport(scanRequestID, reportMIMEType string) (string, error) { + args := mc.Called(scanRequestID, reportMIMEType) + return args.String(0), args.Error(1) +} diff --git a/src/pkg/scan/report/supported_mime_test.go b/src/pkg/scan/report/supported_mime_test.go new file mode 100644 index 000000000..f118e25ff --- /dev/null +++ b/src/pkg/scan/report/supported_mime_test.go @@ -0,0 +1,78 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "encoding/json" + "testing" + "time" + + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/goharbor/harbor/src/pkg/scan/vuln" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// SupportedMimesSuite is a suite to test SupportedMimes. +type SupportedMimesSuite struct { + suite.Suite + + mockData []byte +} + +// TestSupportedMimesSuite is the entry of SupportedMimesSuite. +func TestSupportedMimesSuite(t *testing.T) { + suite.Run(t, new(SupportedMimesSuite)) +} + +// SetupSuite prepares the test suite env. +func (suite *SupportedMimesSuite) SetupSuite() { + rp := vuln.Report{ + GeneratedAt: time.Now().UTC().String(), + Scanner: &v1.Scanner{ + Name: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + }, + Severity: vuln.High, + Vulnerabilities: []*vuln.VulnerabilityItem{ + { + ID: "2019-0980-0909", + Package: "dpkg", + Version: "0.9.1", + FixVersion: "0.9.2", + Severity: vuln.High, + Description: "mock one", + Links: []string{"https://vuln.com"}, + }, + }, + } + + jsonData, err := json.Marshal(rp) + require.NoError(suite.T(), err) + suite.mockData = jsonData +} + +// TestResolveData tests the ResolveData. +func (suite *SupportedMimesSuite) TestResolveData() { + obj, err := ResolveData(v1.MimeTypeNativeReport, suite.mockData) + require.NoError(suite.T(), err) + require.Condition(suite.T(), func() (success bool) { + rp, ok := obj.(*vuln.Report) + success = ok && rp != nil && rp.Severity == vuln.High + + return + }) +} diff --git a/src/pkg/scan/report/supported_mimes.go b/src/pkg/scan/report/supported_mimes.go new file mode 100644 index 000000000..80bf60aca --- /dev/null +++ b/src/pkg/scan/report/supported_mimes.go @@ -0,0 +1,56 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "encoding/json" + "reflect" + + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/goharbor/harbor/src/pkg/scan/vuln" + "github.com/pkg/errors" +) + +// SupportedMimes indicates what mime types are supported to render at UI end. +var SupportedMimes = map[string]interface{}{ + // The native report type + v1.MimeTypeNativeReport: (*vuln.Report)(nil), +} + +// ResolveData is a helper func to parse the JSON data with the given mime type. +func ResolveData(mime string, jsonData []byte) (interface{}, error) { + if len(jsonData) == 0 { + return nil, errors.New("empty JSON data") + } + + t, ok := SupportedMimes[mime] + if !ok { + return nil, errors.Errorf("report with mime type %s is not supported", mime) + } + + ty := reflect.TypeOf(t) + if ty.Kind() == reflect.Ptr { + ty = ty.Elem() + } + + // New one + rp := reflect.New(ty).Elem().Addr().Interface() + + if err := json.Unmarshal(jsonData, rp); err != nil { + return nil, err + } + + return rp, nil +} diff --git a/src/pkg/scan/rest/auth/api_key_auth.go b/src/pkg/scan/rest/auth/api_key_auth.go new file mode 100644 index 000000000..80f79fd6d --- /dev/null +++ b/src/pkg/scan/rest/auth/api_key_auth.go @@ -0,0 +1,45 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "net/http" + + "github.com/pkg/errors" +) + +// apiKeyAuthorizer authorize by adding a header `X-ScannerAdapter-API-Key` with value "credential" +type apiKeyAuthorizer struct { + typeID string + accessCred string +} + +// Authorize the requests +func (aa *apiKeyAuthorizer) Authorize(req *http.Request) error { + if req != nil && len(aa.accessCred) > 0 { + req.Header.Add(aa.typeID, aa.accessCred) + return nil + } + + return errors.Errorf("%s: %s", aa.typeID, "missing data to authorize request") +} + +// NewAPIKeyAuthorizer news a apiKeyAuthorizer +func NewAPIKeyAuthorizer(accessCred string) Authorizer { + return &apiKeyAuthorizer{ + typeID: APIKey, + accessCred: accessCred, + } +} diff --git a/src/pkg/scan/rest/auth/auth.go b/src/pkg/scan/rest/auth/auth.go new file mode 100644 index 000000000..f4cf29b40 --- /dev/null +++ b/src/pkg/scan/rest/auth/auth.go @@ -0,0 +1,54 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "net/http" + "strings" + + "github.com/pkg/errors" +) + +const ( + authorization = "Authorization" + // Basic ... + Basic = "Basic" + // Bearer ... + Bearer = "Bearer" + // APIKey ... + APIKey = "X-ScannerAdapter-API-Key" +) + +// Authorizer defines operation for authorizing the requests +type Authorizer interface { + Authorize(req *http.Request) error +} + +// GetAuthorizer is a factory method for getting an authorizer based on the given auth type +func GetAuthorizer(auth, cred string) (Authorizer, error) { + switch strings.TrimSpace(auth) { + // No authorizer required + case "": + return NewNoAuth(), nil + case Basic: + return NewBasicAuth(cred), nil + case Bearer: + return NewBearerAuth(cred), nil + case APIKey: + return NewAPIKeyAuthorizer(cred), nil + default: + return nil, errors.Errorf("auth type %s is not supported", auth) + } +} diff --git a/src/pkg/scan/rest/auth/basic_auth.go b/src/pkg/scan/rest/auth/basic_auth.go new file mode 100644 index 000000000..25256c2cf --- /dev/null +++ b/src/pkg/scan/rest/auth/basic_auth.go @@ -0,0 +1,48 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "encoding/base64" + "fmt" + "net/http" + + "github.com/pkg/errors" +) + +// basicAuthorizer authorizes the request by adding `Authorization Basic base64(credential)` header +type basicAuthorizer struct { + typeID string + accessCred string +} + +// Authorize requests +func (ba *basicAuthorizer) Authorize(req *http.Request) error { + if len(ba.accessCred) == 0 { + return errors.Errorf("%s:%s", ba.typeID, "missing access credential") + } + + if req != nil && len(ba.accessCred) > 0 { + data := base64.StdEncoding.EncodeToString([]byte(ba.accessCred)) + req.Header.Add(authorization, fmt.Sprintf("%s %s", ba.typeID, data)) + } + + return errors.Errorf("%s: %s", ba.typeID, "missing data to authorize request") +} + +// NewBasicAuth basic authorizer +func NewBasicAuth(accessCred string) Authorizer { + return &basicAuthorizer{Basic, accessCred} +} diff --git a/src/pkg/scan/rest/auth/bearer_auth.go b/src/pkg/scan/rest/auth/bearer_auth.go new file mode 100644 index 000000000..d7b8f383c --- /dev/null +++ b/src/pkg/scan/rest/auth/bearer_auth.go @@ -0,0 +1,40 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "fmt" + "net/http" +) + +// bearerAuthorizer authorizes the request by adding `Authorization Bearer credential` header +type bearerAuthorizer struct { + typeID string + accessCred string +} + +// Authorize requests +func (ba *bearerAuthorizer) Authorize(req *http.Request) error { + if req != nil && len(ba.accessCred) > 0 { + req.Header.Add(authorization, fmt.Sprintf("%s %s", ba.typeID, ba.accessCred)) + } + + return nil +} + +// NewBearerAuth create bearer authorizer +func NewBearerAuth(accessCred string) Authorizer { + return &bearerAuthorizer{Bearer, accessCred} +} diff --git a/src/pkg/scan/scanner/api/scan.go b/src/pkg/scan/rest/auth/no_auth.go similarity index 58% rename from src/pkg/scan/scanner/api/scan.go rename to src/pkg/scan/rest/auth/no_auth.go index a8b1a47d4..c41375a4b 100644 --- a/src/pkg/scan/scanner/api/scan.go +++ b/src/pkg/scan/rest/auth/no_auth.go @@ -12,24 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package api +package auth import ( - dscan "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scan" - "github.com/goharbor/harbor/src/pkg/scan/scanner/scan" + "net/http" ) -// Scan ... -func (bc *basicController) Scan(artifact *scan.Artifact) error { +// noAuth is created to handle the no authorization case which is acceptable +type noAuth struct{} + +// Authorize the incoming request +func (na *noAuth) Authorize(req *http.Request) error { + // Do nothing return nil } -// GetReport ... -func (bc *basicController) GetReport(artifact *scan.Artifact) ([]*dscan.Report, error) { - return nil, nil -} - -// GetScanLog ... -func (bc *basicController) GetScanLog(digest string) ([]byte, error) { - return nil, nil +// NewNoAuth creates a noAuth authorizer +func NewNoAuth() Authorizer { + return &noAuth{} } diff --git a/src/pkg/scan/rest/v1/client.go b/src/pkg/scan/rest/v1/client.go new file mode 100644 index 000000000..9dfe9711d --- /dev/null +++ b/src/pkg/scan/rest/v1/client.go @@ -0,0 +1,278 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "io/ioutil" + "net" + "net/http" + "strconv" + "time" + + "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scan/rest/auth" + "github.com/pkg/errors" +) + +const ( + // defaultRefreshInterval is the default interval with seconds of refreshing report + defaultRefreshInterval = 5 + // refreshAfterHeader provides the refresh interval value + refreshAfterHeader = "Refresh-After" +) + +// Client defines the methods to access the adapter services that +// implement the REST API specs +type Client interface { + // GetMetadata gets the metadata of the given scanner + // + // Returns: + // *ScannerAdapterMetadata : metadata of the given scanner + // error : non nil error if any errors occurred + GetMetadata() (*ScannerAdapterMetadata, error) + + // SubmitScan initiates a scanning of the given artifact. + // Returns `nil` if the request was accepted, a non `nil` error otherwise. + // + // Arguments: + // req *ScanRequest : request including the registry and artifact data + // + // Returns: + // *ScanResponse : response with UUID for tracking the scan results + // error : non nil error if any errors occurred + SubmitScan(req *ScanRequest) (*ScanResponse, error) + + // GetScanReport gets the scan result for the corresponding ScanRequest identifier. + // Note that this is a blocking method which either returns a non `nil` scan report or error. + // A caller is supposed to cast the returned interface{} to a structure that corresponds + // to the specified MIME type. + // + // Arguments: + // scanRequestID string : the ID of the scan submitted before + // reportMIMEType string : the report mime type + // Returns: + // string : the scan report of the given artifact + // error : non nil error if any errors occurred + GetScanReport(scanRequestID, reportMIMEType string) (string, error) +} + +// basicClient is default implementation of the Client interface +type basicClient struct { + httpClient *http.Client + spec *Spec + authorizer auth.Authorizer +} + +// NewClient news a basic client +func NewClient(r *scanner.Registration) (Client, error) { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: r.SkipCertVerify, + }, + } + + authorizer, err := auth.GetAuthorizer(r.Auth, r.AccessCredential) + if err != nil { + return nil, errors.Wrap(err, "new v1 client") + } + + return &basicClient{ + httpClient: &http.Client{ + Transport: transport, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + spec: NewSpec(r.URL), + authorizer: authorizer, + }, nil +} + +// GetMetadata ... +func (c *basicClient) GetMetadata() (*ScannerAdapterMetadata, error) { + def := c.spec.Metadata() + + request, err := http.NewRequest(http.MethodGet, def.URL, nil) + if err != nil { + return nil, errors.Wrap(err, "v1 client: get metadata") + } + + // Resolve header + def.Resolver(request) + + // Send request + respData, err := c.send(request, generalResponseHandler(http.StatusOK)) + if err != nil { + return nil, errors.Wrap(err, "v1 client: get metadata") + } + + // Unmarshal data + meta := &ScannerAdapterMetadata{} + if err := json.Unmarshal(respData, meta); err != nil { + return nil, errors.Wrap(err, "v1 client: get metadata") + } + + return meta, nil +} + +// SubmitScan ... +func (c *basicClient) SubmitScan(req *ScanRequest) (*ScanResponse, error) { + if req == nil { + return nil, errors.New("nil request") + } + + data, err := json.Marshal(req) + if err != nil { + return nil, errors.Wrap(err, "v1 client: submit scan") + } + + def := c.spec.SubmitScan() + request, err := http.NewRequest(http.MethodPost, def.URL, bytes.NewReader(data)) + if err != nil { + return nil, errors.Wrap(err, "v1 client: submit scan") + } + + respData, err := c.send(request, generalResponseHandler(http.StatusCreated)) + if err != nil { + return nil, errors.Wrap(err, "v1 client: submit scan") + } + + resp := &ScanResponse{} + if err := json.Unmarshal(respData, resp); err != nil { + return nil, errors.Wrap(err, "v1 client: submit scan") + } + + return resp, nil +} + +// GetScanReport ... +func (c *basicClient) GetScanReport(scanRequestID, reportMIMEType string) (string, error) { + if len(scanRequestID) == 0 { + return "", errors.New("empty scan request ID") + } + + if len(reportMIMEType) == 0 { + return "", errors.New("missing report mime type") + } + + def := c.spec.GetScanReport(scanRequestID, reportMIMEType) + + req, err := http.NewRequest(http.MethodGet, def.URL, nil) + if err != nil { + return "", errors.Wrap(err, "v1 client: get scan report") + } + + respData, err := c.send(req, reportResponseHandler()) + if err != nil { + // This error should not be wrapped + return "", err + } + + return string(respData), nil +} + +func (c *basicClient) send(req *http.Request, h responseHandler) ([]byte, error) { + if c.authorizer != nil { + if err := c.authorizer.Authorize(req); err != nil { + return nil, errors.Wrap(err, "authorization") + } + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + + defer func() { + if err := resp.Body.Close(); err != nil { + // Just logged + logger.Errorf("close response body error: %s", err) + } + }() + + return h(resp.StatusCode, resp) +} + +// responseHandlerFunc is a handler func template for handling the http response data, +// especially the error part. +type responseHandler func(code int, resp *http.Response) ([]byte, error) + +// generalResponseHandler create a general response handler to cover the common cases. +func generalResponseHandler(expectedCode int) responseHandler { + return func(code int, resp *http.Response) ([]byte, error) { + return generalRespHandlerFunc(expectedCode, code, resp) + } +} + +// reportResponseHandler creates response handler for get report special case. +func reportResponseHandler() responseHandler { + return func(code int, resp *http.Response) ([]byte, error) { + if code == http.StatusFound { + // Set default + retryAfter := defaultRefreshInterval // seconds + // Read `retry after` info from header + v := resp.Header.Get(refreshAfterHeader) + if len(v) > 0 { + if i, err := strconv.ParseInt(v, 10, 8); err == nil { + retryAfter = int(i) + } else { + // log error + logger.Errorf("Parse `%s` error: %s", refreshAfterHeader, err) + } + } + + return nil, &ReportNotReadyError{RetryAfter: retryAfter} + } + + return generalRespHandlerFunc(http.StatusOK, code, resp) + } +} + +// generalRespHandlerFunc is a handler to cover the general cases +func generalRespHandlerFunc(expectedCode, code int, resp *http.Response) ([]byte, error) { + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if code != expectedCode { + if len(buf) > 0 { + // Try to read error response + eResp := &ErrorResponse{ + Err: &Error{}, + } + if err := json.Unmarshal(buf, eResp); err == nil { + return nil, eResp + } + } + + return nil, errors.Errorf("unexpected status code: %d, response: %s", code, string(buf)) + } + + return buf, nil +} diff --git a/src/pkg/scan/rest/v1/client_pool.go b/src/pkg/scan/rest/v1/client_pool.go new file mode 100644 index 000000000..0084b0d76 --- /dev/null +++ b/src/pkg/scan/rest/v1/client_pool.go @@ -0,0 +1,128 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "encoding/base64" + "fmt" + "sync" + + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + "github.com/pkg/errors" +) + +// DefaultClientPool is a default client pool. +var DefaultClientPool = NewClientPool() + +// ClientPool defines operations for the client pool which provides v1 client cache. +type ClientPool interface { + // Get a v1 client interface for the specified registration. + // + // Arguments: + // r *scanner.Registration : registration for client connecting to + // + // Returns: + // Client : v1 client + // error : non nil error if any errors occurred + Get(r *scanner.Registration) (Client, error) +} + +// basicClientPool is default implementation of client pool interface. +type basicClientPool struct { + pool *sync.Map +} + +// NewClientPool news a basic client pool. +func NewClientPool() ClientPool { + return &basicClientPool{ + pool: &sync.Map{}, + } +} + +// Get client for the specified registration. +// So far, there will not be too many scanner registrations. An then +// no need to do client instance clear work. +// If one day, we have to clear unactivated client instances in the pool, +// add the following func after the first time initializing the client. +// pool item represents the client with a timestamp of last accessed. +// +// type poolItem struct { +// c Client +// timestamp time.Time +// } +// +// func (bcp *basicClientPool) deadCheck(key string, item *poolItem) { +// // Run in a separate goroutine +// go func() { +// // As we do not have a global context, let's watch the system signal to +// // exit the goroutine correctly. +// sig := make(chan os.Signal, 1) +// signal.Notify(sig, os.Interrupt, syscall.SIGTERM, os.Kill) +// +// tk := time.NewTicker(bcp.config.DeadCheckInterval) +// defer tk.Stop() +// +// for { +// select { +// case t := <-tk.C: +// if item.timestamp.Add(bcp.config.ExpireTime).Before(t.UTC()) { +// // Expired +// bcp.pool.Delete(key) +// return +// } +// case <-sig: +// // Terminated by system +// return +// } +// } +// }() +// } +func (bcp *basicClientPool) Get(r *scanner.Registration) (Client, error) { + if r == nil { + return nil, errors.New("nil scanner registration") + } + + if err := r.Validate(true); err != nil { + return nil, errors.Wrap(err, "client pool: get") + } + + k := key(r) + + c, ok := bcp.pool.Load(k) + if !ok { + nc, err := NewClient(r) + if err != nil { + return nil, errors.Wrap(err, "client pool: get") + } + + // Cache it + bcp.pool.Store(k, nc) + c = nc + } + + return c.(Client), nil +} + +func key(r *scanner.Registration) string { + raw := fmt.Sprintf("%s:%s:%s:%s:%v", + r.UUID, + r.URL, + r.Auth, + r.AccessCredential, + r.SkipCertVerify, + ) + + return base64.StdEncoding.EncodeToString([]byte(raw)) +} diff --git a/src/pkg/scan/rest/v1/client_pool_test.go b/src/pkg/scan/rest/v1/client_pool_test.go new file mode 100644 index 000000000..95e316d41 --- /dev/null +++ b/src/pkg/scan/rest/v1/client_pool_test.go @@ -0,0 +1,69 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "fmt" + "testing" + + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scan/rest/auth" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// ClientPoolTestSuite is a test suite to test the client pool. +type ClientPoolTestSuite struct { + suite.Suite + + pool ClientPool +} + +// TestClientPool is the entry of ClientPoolTestSuite. +func TestClientPool(t *testing.T) { + suite.Run(t, new(ClientPoolTestSuite)) +} + +// SetupSuite sets up test suite env. +func (suite *ClientPoolTestSuite) SetupSuite() { + suite.pool = NewClientPool() +} + +// TestClientPoolGet tests the get method of client pool. +func (suite *ClientPoolTestSuite) TestClientPoolGet() { + r := &scanner.Registration{ + ID: 1, + Name: "TestClientPoolGet", + UUID: "uuid", + URL: "http://a.b.c", + Auth: auth.Basic, + AccessCredential: "u:p", + SkipCertVerify: false, + } + + client1, err := suite.pool.Get(r) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), client1) + + p1 := fmt.Sprintf("%p", client1.(*basicClient)) + + client2, err := suite.pool.Get(r) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), client2) + + p2 := fmt.Sprintf("%p", client2.(*basicClient)) + assert.Equal(suite.T(), p1, p2) +} diff --git a/src/pkg/scan/rest/v1/client_test.go b/src/pkg/scan/rest/v1/client_test.go new file mode 100644 index 000000000..84767555b --- /dev/null +++ b/src/pkg/scan/rest/v1/client_test.go @@ -0,0 +1,197 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// ClientTestSuite tests the v1 client +type ClientTestSuite struct { + suite.Suite + + testServer *httptest.Server + client Client +} + +// TestClient is the entry of ClientTestSuite +func TestClient(t *testing.T) { + suite.Run(t, new(ClientTestSuite)) +} + +// SetupSuite prepares the test suite env +func (suite *ClientTestSuite) SetupSuite() { + suite.testServer = httptest.NewServer(&mockHandler{}) + r := &scanner.Registration{ + ID: 1000, + UUID: "uuid", + Name: "TestClient", + URL: suite.testServer.URL, + SkipCertVerify: true, + } + + c, err := NewClient(r) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), c) + + suite.client = c +} + +// TestClientMetadata tests the metadata of the client +func (suite *ClientTestSuite) TestClientMetadata() { + m, err := suite.client.GetMetadata() + require.NoError(suite.T(), err) + require.NotNil(suite.T(), m) + + assert.Equal(suite.T(), m.Scanner.Name, "Clair") +} + +// TestClientSubmitScan tests the scan submission of client +func (suite *ClientTestSuite) TestClientSubmitScan() { + res, err := suite.client.SubmitScan(&ScanRequest{}) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), res) + + assert.Equal(suite.T(), res.ID, "123456789") +} + +// TestClientGetScanReportError tests getting report failed +func (suite *ClientTestSuite) TestClientGetScanReportError() { + _, err := suite.client.GetScanReport("id1", MimeTypeNativeReport) + require.Error(suite.T(), err) + assert.Condition(suite.T(), func() (success bool) { + success = strings.Index(err.Error(), "error") != -1 + return + }) +} + +// TestClientGetScanReport tests getting report +func (suite *ClientTestSuite) TestClientGetScanReport() { + res, err := suite.client.GetScanReport("id2", MimeTypeNativeReport) + require.NoError(suite.T(), err) + require.NotEmpty(suite.T(), res) +} + +// TestClientGetScanReportNotReady tests the case that the report is not ready +func (suite *ClientTestSuite) TestClientGetScanReportNotReady() { + _, err := suite.client.GetScanReport("id3", MimeTypeNativeReport) + require.Error(suite.T(), err) + require.Condition(suite.T(), func() (success bool) { + _, success = err.(*ReportNotReadyError) + return + }) + assert.Equal(suite.T(), 10, err.(*ReportNotReadyError).RetryAfter) +} + +// TearDownSuite clears the test suite env +func (suite *ClientTestSuite) TearDownSuite() { + suite.testServer.Close() +} + +type mockHandler struct{} + +// ServeHTTP ... +func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch r.RequestURI { + case "/metadata": + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusForbidden) + return + } + m := &ScannerAdapterMetadata{ + Scanner: &Scanner{ + Name: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + }, + Capabilities: &ScannerCapability{ + ConsumesMimeTypes: []string{ + MimeTypeOCIArtifact, + MimeTypeDockerArtifact, + }, + ProducesMimeTypes: []string{ + MimeTypeNativeReport, + MimeTypeRawReport, + }, + }, + Properties: ScannerProperties{ + "extra": "testing", + }, + } + data, _ := json.Marshal(m) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(data) + break + case "/scan": + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusForbidden) + return + } + + res := &ScanResponse{} + res.ID = "123456789" + + data, _ := json.Marshal(res) + + w.WriteHeader(http.StatusCreated) + _, _ = w.Write(data) + break + case "/scan/id1/report": + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusForbidden) + return + } + + e := &ErrorResponse{ + &Error{ + Message: "error", + }, + } + + data, _ := json.Marshal(e) + + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write(data) + break + case "/scan/id2/report": + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + break + case "/scan/id3/report": + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusForbidden) + return + } + + w.Header().Add(refreshAfterHeader, fmt.Sprintf("%d", 10)) + w.Header().Add("Location", "/scan/id3/report") + w.WriteHeader(http.StatusFound) + break + } +} diff --git a/src/pkg/scan/rest/v1/models.go b/src/pkg/scan/rest/v1/models.go new file mode 100644 index 000000000..47ded4eb1 --- /dev/null +++ b/src/pkg/scan/rest/v1/models.go @@ -0,0 +1,173 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "encoding/json" + "fmt" + + "github.com/pkg/errors" +) + +// Scanner represents metadata of a Scanner Adapter which allow Harbor to lookup a scanner capable of +// scanning a given Artifact stored in its registry and making sure that it can interpret a +// returned result. +type Scanner struct { + // The name of the scanner. + Name string `json:"name"` + // The name of the scanner's provider. + Vendor string `json:"vendor"` + // The version of the scanner. + Version string `json:"version"` +} + +// ScannerCapability consists of the set of recognized artifact MIME types and the set of scanner +// report MIME types. For example, a scanner capable of analyzing Docker images and producing +// a vulnerabilities report recognizable by Harbor web console might be represented with the +// following capability: +// - consumes MIME types: +// -- application/vnd.oci.image.manifest.v1+json +// -- application/vnd.docker.distribution.manifest.v2+json +// - produces MIME types +// -- application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0 +// -- application/vnd.scanner.adapter.vuln.report.raw +type ScannerCapability struct { + // The set of MIME types of the artifacts supported by the scanner to produce the reports + // specified in the "produces_mime_types". A given mime type should only be present in one + // capability item. + ConsumesMimeTypes []string `json:"consumes_mime_types"` + // The set of MIME types of reports generated by the scanner for the consumes_mime_types of + // the same capability record. + ProducesMimeTypes []string `json:"produces_mime_types"` +} + +// ScannerProperties is a set of custom properties that can further describe capabilities of a given scanner. +type ScannerProperties map[string]string + +// ScannerAdapterMetadata represents metadata of a Scanner Adapter which allows Harbor to lookup +// a scanner capable of scanning a given Artifact stored in its registry and making sure that it +// can interpret a returned result. +type ScannerAdapterMetadata struct { + Scanner *Scanner `json:"scanner"` + Capabilities *ScannerCapability `json:"capabilities"` + Properties ScannerProperties `json:"properties"` +} + +// Artifact represents an artifact stored in Registry. +type Artifact struct { + // The full name of a Harbor repository containing the artifact, including the namespace. + // For example, `library/oracle/nosql`. + Repository string `json:"repository"` + // The artifact's digest, consisting of an algorithm and hex portion. + // For example, `sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b`, + // represents sha256 based digest. + Digest string `json:"digest"` + // The mime type of the scanned artifact + MimeType string `json:"mime_type"` +} + +// Registry represents Registry connection settings. +type Registry struct { + // A base URL of the Docker Registry v2 API exposed by Harbor. + URL string `json:"url"` + // An optional value of the HTTP Authorization header sent with each request to the Docker Registry v2 API. + // For example, `Bearer: JWTTOKENGOESHERE`. + Authorization string `json:"authorization"` +} + +// ScanRequest represents a structure that is sent to a Scanner Adapter to initiate artifact scanning. +// Conducts all the details required to pull the artifact from a Harbor registry. +type ScanRequest struct { + // Connection settings for the Docker Registry v2 API exposed by Harbor. + Registry *Registry `json:"registry"` + // Artifact to be scanned. + Artifact *Artifact `json:"artifact"` +} + +// FromJSON parses ScanRequest from json data +func (s *ScanRequest) FromJSON(jsonData string) error { + if len(jsonData) == 0 { + return errors.New("empty json data to parse") + } + + return json.Unmarshal([]byte(jsonData), s) +} + +// ToJSON marshals ScanRequest to JSON data +func (s *ScanRequest) ToJSON() (string, error) { + data, err := json.Marshal(s) + if err != nil { + return "", err + } + + return string(data), nil +} + +// Validate ScanRequest +func (s *ScanRequest) Validate() error { + if s.Registry == nil || + len(s.Registry.URL) == 0 || + len(s.Registry.Authorization) == 0 { + return errors.New("scan request: invalid registry") + } + + if s.Artifact == nil || + len(s.Artifact.Digest) == 0 || + len(s.Artifact.Repository) == 0 || + len(s.Artifact.MimeType) == 0 { + return errors.New("scan request: invalid artifact") + } + + return nil +} + +// ScanResponse represents the response returned by the scanner adapter after scan request successfully +// submitted. +type ScanResponse struct { + // e.g: 3fa85f64-5717-4562-b3fc-2c963f66afa6 + ID string `json:"id"` +} + +// ErrorResponse contains error message when requests are not correctly handled. +type ErrorResponse struct { + // Error object + Err *Error `json:"error"` +} + +// Error message +type Error struct { + // Message of the error + Message string `json:"message"` +} + +// Error for ErrorResponse +func (er *ErrorResponse) Error() string { + if er.Err != nil { + return er.Err.Message + } + + return "nil error" +} + +// ReportNotReadyError is an error to indicate the scan report is not ready +type ReportNotReadyError struct { + // Seconds for next retry with seconds + RetryAfter int +} + +// Error for ReportNotReadyError +func (rnr *ReportNotReadyError) Error() string { + return fmt.Sprintf("report is not ready yet, retry after %d", rnr.RetryAfter) +} diff --git a/src/pkg/scan/rest/v1/spec.go b/src/pkg/scan/rest/v1/spec.go new file mode 100644 index 000000000..d03ddc29b --- /dev/null +++ b/src/pkg/scan/rest/v1/spec.go @@ -0,0 +1,109 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "fmt" + "net/http" + "strings" +) + +const ( + // HTTPAcceptHeader represents the HTTP accept header + HTTPAcceptHeader = "Accept" + // HTTPContentType represents the HTTP content-type header + HTTPContentType = "Content-Type" + // MimeTypeOCIArtifact defines the mime type for OCI artifact + MimeTypeOCIArtifact = "application/vnd.oci.image.manifest.v1+json" + // MimeTypeDockerArtifact defines the mime type for docker artifact + MimeTypeDockerArtifact = "application/vnd.docker.distribution.manifest.v2+json" + // MimeTypeNativeReport defines the mime type for native report + MimeTypeNativeReport = "application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0" + // MimeTypeRawReport defines the mime type for raw report + MimeTypeRawReport = "application/vnd.scanner.adapter.vuln.report.raw" + // MimeTypeAdapterMeta defines the mime type for adapter metadata + MimeTypeAdapterMeta = "application/vnd.scanner.adapter.metadata+json; version=1.0" + // MimeTypeScanRequest defines the mime type for scan request + MimeTypeScanRequest = "application/vnd.scanner.adapter.scan.request+json; version=1.0" + // MimeTypeScanResponse defines the mime type for scan response + MimeTypeScanResponse = "application/vnd.scanner.adapter.scan.response+json; version=1.0" +) + +// RequestResolver is a function template to modify the API request, e.g: add headers +type RequestResolver func(req *http.Request) + +// Definition for API +type Definition struct { + // URL of the API + URL string + // Resolver fro the request + Resolver RequestResolver +} + +// Spec of the API +// Contains URL and possible headers. +type Spec struct { + baseRoute string +} + +// NewSpec news V1 spec +func NewSpec(base string) *Spec { + baseRoute := "http://localhost" + + if len(base) > 0 { + if strings.HasSuffix(base, "/") { + baseRoute = base[:len(base)-1] + } else { + baseRoute = base + } + } + + return &Spec{ + baseRoute: baseRoute, + } +} + +// Metadata API +func (s *Spec) Metadata() Definition { + return Definition{ + URL: fmt.Sprintf("%s%s", s.baseRoute, "/metadata"), + Resolver: func(req *http.Request) { + req.Header.Add(HTTPAcceptHeader, MimeTypeAdapterMeta) + }, + } +} + +// SubmitScan API +func (s *Spec) SubmitScan() Definition { + return Definition{ + URL: fmt.Sprintf("%s%s", s.baseRoute, "/scan"), + Resolver: func(req *http.Request) { + req.Header.Add(HTTPContentType, MimeTypeScanRequest) + req.Header.Add(HTTPAcceptHeader, MimeTypeScanResponse) + }, + } +} + +// GetScanReport API +func (s *Spec) GetScanReport(scanReqID string, mimeType string) Definition { + path := fmt.Sprintf("/scan/%s/report", scanReqID) + + return Definition{ + URL: fmt.Sprintf("%s%s", s.baseRoute, path), + Resolver: func(req *http.Request) { + req.Header.Add(HTTPAcceptHeader, mimeType) + }, + } +} diff --git a/src/pkg/scan/scanner/manager.go b/src/pkg/scan/scanner/manager.go index 3387a8fb7..c6421fe51 100644 --- a/src/pkg/scan/scanner/manager.go +++ b/src/pkg/scan/scanner/manager.go @@ -16,7 +16,7 @@ package scanner import ( "github.com/goharbor/harbor/src/pkg/q" - "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" "github.com/google/uuid" "github.com/pkg/errors" ) @@ -59,7 +59,7 @@ func New() Manager { // Create ... func (bm *basicManager) Create(registration *scanner.Registration) (string, error) { if registration == nil { - return "", errors.New("nil endpoint to create") + return "", errors.New("nil registration to create") } // Inject new UUID @@ -92,11 +92,11 @@ func (bm *basicManager) Get(registrationUUID string) (*scanner.Registration, err // Update ... func (bm *basicManager) Update(registration *scanner.Registration) error { if registration == nil { - return errors.New("nil endpoint to update") + return errors.New("nil registration to update") } if err := registration.Validate(true); err != nil { - return errors.Wrap(err, "update endpoint") + return errors.Wrap(err, "update registration") } return scanner.UpdateRegistration(registration) diff --git a/src/pkg/scan/scanner/manager_test.go b/src/pkg/scan/scanner/manager_test.go index 0cee77117..169a216f0 100644 --- a/src/pkg/scan/scanner/manager_test.go +++ b/src/pkg/scan/scanner/manager_test.go @@ -19,7 +19,7 @@ import ( "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/pkg/q" - "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -48,9 +48,6 @@ func (suite *BasicManagerTestSuite) SetupSuite() { Name: "forUT", Description: "sample registration", URL: "https://sample.scanner.com", - Adapter: "Clair", - Version: "0.1.0", - Vendor: "Harbor", } uid, err := suite.mgr.Create(r) diff --git a/src/pkg/scan/scanner/scan/controller.go b/src/pkg/scan/scanner/scan/controller.go deleted file mode 100644 index e7feb622f..000000000 --- a/src/pkg/scan/scanner/scan/controller.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scan - -import "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scan" - -// Options object for the scan action -type Options struct{} - -// Option for scan action -type Option interface { - // Apply option to the passing in options - Apply(options *Options) error -} - -// Controller defines operations for scan controlling -type Controller interface { - // Scan the given artifact - // - // Arguments: - // artifact *res.Artifact : artifact to be scanned - // - // Returns: - // error : non nil error if any errors occurred - Scan(artifact *Artifact, options ...Option) error - - // GetReport gets the reports for the given artifact identified by the digest - // - // Arguments: - // artifact *res.Artifact : the scanned artifact - // - // Returns: - // []*scan.Report : scan results by different scanner vendors - // error : non nil error if any errors occurred - GetReport(artifact *Artifact) ([]*scan.Report, error) -} diff --git a/src/pkg/scan/scanner/scan/models.go b/src/pkg/scan/scanner/scan/models.go deleted file mode 100644 index 7bb67b909..000000000 --- a/src/pkg/scan/scanner/scan/models.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scan - -// Artifact represents an artifact stored in Registry. -type Artifact struct { - // The full name of a Harbor repository containing the artifact, including the namespace. - // For example, `library/oracle/nosql`. - Repository string - // The artifact's digest, consisting of an algorithm and hex portion. - // For example, `sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b`, - // represents sha256 based digest. - Digest string - // The mime type of the scanned artifact - MimeType string -} - -// Registry represents Registry connection settings. -type Registry struct { - // A base URL of the Docker Registry v2 API exposed by Harbor. - URL string - // An optional value of the HTTP Authorization header sent with each request to the Docker Registry v2 API. - // For example, `Bearer: JWTTOKENGOESHERE`. - Authorization string -} - -// Request represents a structure that is sent to a Scanner Adapter to initiate artifact scanning. -// Conducts all the details required to pull the artifact from a Harbor registry. -type Request struct { - // Connection settings for the Docker Registry v2 API exposed by Harbor. - Registry *Registry - // Artifact to be scanned. - Artifact *Artifact -} diff --git a/src/pkg/scan/vuln/report.go b/src/pkg/scan/vuln/report.go new file mode 100644 index 000000000..7510f7cb6 --- /dev/null +++ b/src/pkg/scan/vuln/report.go @@ -0,0 +1,58 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vuln + +import ( + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" +) + +// Report model for vulnerability scan +type Report struct { + // Time of generating this report + GeneratedAt string `json:"generated_at"` + // Scanner of generating this report + Scanner *v1.Scanner `json:"scanner"` + // A standard scale for measuring the severity of a vulnerability. + Severity Severity `json:"severity"` + // Vulnerability list + Vulnerabilities []*VulnerabilityItem `json:"vulnerabilities"` +} + +// VulnerabilityItem represents one found vulnerability +type VulnerabilityItem struct { + // The unique identifier of the vulnerability. + // e.g: CVE-2017-8283 + ID string `json:"id"` + // An operating system or software dependency package containing the vulnerability. + // e.g: dpkg + Package string `json:"package"` + // The version of the package containing the vulnerability. + // e.g: 1.17.27 + Version string `json:"version"` + // The version of the package containing the fix if available. + // e.g: 1.18.0 + FixVersion string `json:"fix_version"` + // A standard scale for measuring the severity of a vulnerability. + Severity Severity `json:"severity"` + // example: dpkg-source in dpkg 1.3.0 through 1.18.23 is able to use a non-GNU patch program + // and does not offer a protection mechanism for blank-indented diff hunks, which allows remote + // attackers to conduct directory traversal attacks via a crafted Debian source package, as + // demonstrated by using of dpkg-source on NetBSD. + Description string `json:"description"` + // The list of link to the upstream database with the full description of the vulnerability. + // Format: URI + // e.g: List [ "https://security-tracker.debian.org/tracker/CVE-2017-8283" ] + Links []string +} diff --git a/src/pkg/scan/vuln/severity.go b/src/pkg/scan/vuln/severity.go new file mode 100644 index 000000000..3d1df84f5 --- /dev/null +++ b/src/pkg/scan/vuln/severity.go @@ -0,0 +1,39 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vuln + +const ( + // Unknown - either a security problem that has not been assigned to a priority yet or + // a priority that the scanner did not recognize. + Unknown Severity = "Unknown" + // Low - a security problem, but is hard to exploit due to environment, requires a + // user-assisted attack, a small install base, or does very little damage. + Low Severity = "Low" + // Negligible - technically a security problem, but is only theoretical in nature, requires + // a very special situation, has almost no install base, or does no real damage. + Negligible Severity = "Negligible" + // Medium - a real security problem, and is exploitable for many people. Includes network + // daemon denial of service attacks, cross-site scripting, and gaining user privileges. + Medium Severity = "Medium" + // High - a real problem, exploitable for many people in a default installation. Includes + // serious remote denial of service, local root privilege escalations, or data loss. + High Severity = "High" + // Critical - a world-burning problem, exploitable for nearly all people in a default installation. + // Includes remote root privilege escalations, or massive data loss. + Critical Severity = "Critical" +) + +// Severity is a standard scale for measuring the severity of a vulnerability. +type Severity string