Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-22 18:25:56 +01:00
Code Issues Projects Releases Wiki Activity

[stage2]support pluggable scanner

Browse Source 
- add scanner rest API v1 spec
- implement v1 client which is used to talk to scanner adapter
- adjust data/orm models
- adjust code package structure

Signed-off-by: Steven Zou <szou@vmware.com>

- implement scan client which is used to talk to scanner adapter
- implement scan job which take the work of communicating with scanner
- update scanner mgmt API routes
- add corresponding UT cases
This commit is contained in:
Steven Zou 2019-09-19 19:15:37 +08:00
parent c360e71d51
commit 0c19eba8c2
36 changed files with 2221 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
make/migrations/postgresql/0011_1.10.0_schema.up.sql
View File

@ -8,9 +8,6 @@ CREATE TABLE scanner_registration
description VARCHAR(1024) NULL,
auth VARCHAR(16) NOT NULL,
access_cred VARCHAR(512) NULL,
adapter VARCHAR(128) NOT NULL,
vendor VARCHAR(128) NOT NULL,
version VARCHAR(32) NOT NULL,
disabled BOOLEAN NOT NULL DEFAULT FALSE,
is_default BOOLEAN NOT NULL DEFAULT FALSE,
skip_cert_verify BOOLEAN NOT NULL DEFAULT FALSE,
@ -18,17 +15,18 @@ CREATE TABLE scanner_registration
update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
/*Table for keeping the scanner report. The report details are stored as JSONB*/
CREATE TABLE scanner_report
/*Table for keeping the scan report. The report details are stored as JSON*/
CREATE TABLE scan_report
(
id SERIAL PRIMARY KEY NOT NULL,
digest VARCHAR(256) NOT NULL,
registration_id VARCHAR(64) NOT NULL,
registration_uuid VARCHAR(64) NOT NULL,
mime_type VARCHAR(256) NOT NULL,
job_id VARCHAR(32),
status VARCHAR(16) NOT NULL,
status_code INTEGER DEFAULT 0,
report JSON,
start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
end_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE(digest, registration_id)
UNIQUE(digest, registration_uuid, mime_type)
)

2
src/core/api/harborapi_test.go
View File

@ -209,7 +209,7 @@ func init() {
// Add routes for plugin scanner management
scannerAPI := &ScannerAPI{}
beego.Router("/api/scanners", scannerAPI, "post:Create;get:List")
beego.Router("/api/scanners/:uid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault")
beego.Router("/api/scanners/:uuid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault")
// Add routes for project level scanner
beego.Router("/api/projects/:pid([0-9]+)/scanner", scannerAPI, "get:GetProjectScanner;put:SetProjectScanner")

14
src/core/api/plug_scanners.go → src/core/api/scanners.go
View File

@ -19,8 +19,8 @@ import (
"net/http"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/scanner/api"
"github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner"
s "github.com/goharbor/harbor/src/pkg/scan/api/scanner"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/pkg/errors"
)
@ -30,7 +30,7 @@ type ScannerAPI struct {
BaseController
// Controller for the plug scanners
c api.Controller
c s.Controller
}
// Prepare sth. for the subsequent actions
@ -50,7 +50,7 @@ func (sa *ScannerAPI) Prepare() {
}
// Use the default controller
sa.c = api.DefaultController
sa.c = s.DefaultController
}
// Get the specified scanner
@ -192,7 +192,7 @@ func (sa *ScannerAPI) Update() {
// Delete the scanner
func (sa *ScannerAPI) Delete() {
uid := sa.GetStringFromPath(":uid")
uid := sa.GetStringFromPath(":uuid")
if len(uid) == 0 {
sa.SendBadRequestError(errors.New("missing uid"))
return
@ -216,7 +216,7 @@ func (sa *ScannerAPI) Delete() {
// SetAsDefault sets the given registration as default one
func (sa *ScannerAPI) SetAsDefault() {
uid := sa.GetStringFromPath(":uid")
uid := sa.GetStringFromPath(":uuid")
if len(uid) == 0 {
sa.SendBadRequestError(errors.New("missing uid"))
return
@ -293,7 +293,7 @@ func (sa *ScannerAPI) SetProjectScanner() {
// get the specified scanner
func (sa *ScannerAPI) get() *scanner.Registration {
uid := sa.GetStringFromPath(":uid")
uid := sa.GetStringFromPath(":uuid")
if len(uid) == 0 {
sa.SendBadRequestError(errors.New("missing uid"))
return nil

69
src/core/api/plug_scanners_test.go → src/core/api/scanners_test.go
View File

@ -20,10 +20,8 @@ import (
"testing"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/scanner/api"
dscan "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/scanner/scan"
sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
@ -38,7 +36,7 @@ const (
type ScannerAPITestSuite struct {
suite.Suite
originC api.Controller
originC sc.Controller
mockC *MockScannerAPIController
}
@ -49,9 +47,9 @@ func TestScannerAPI(t *testing.T) {
// SetupSuite prepares testing env
func (suite *ScannerAPITestSuite) SetupTest() {
suite.originC = api.DefaultController
suite.originC = sc.DefaultController
m := &MockScannerAPIController{}
api.DefaultController = m
sc.DefaultController = m
suite.mockC = m
}
@ -59,7 +57,7 @@ func (suite *ScannerAPITestSuite) SetupTest() {
// TearDownTest clears test case env
func (suite *ScannerAPITestSuite) TearDownTest() {
// Restore
api.DefaultController = suite.originC
sc.DefaultController = suite.originC
}
// TestScannerAPICreate tests the post request to create new one
@ -108,9 +106,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPIGet() {
Name: "TestScannerAPIGet",
Description: "JUST FOR TEST",
URL: "https://a.b.c",
Adapter: "Clair",
Vendor: "Harbor",
Version: "0.1.0",
}
suite.mockC.On("GetRegistration", "uuid").Return(res, nil)
@ -133,9 +128,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPICreate() {
Name: "TestScannerAPICreate",
Description: "JUST FOR TEST",
URL: "https://a.b.c",
Adapter: "Clair",
Vendor: "Harbor",
Version: "0.1.0",
}
suite.mockQuery(r)
@ -170,9 +162,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPIList() {
Name: "TestScannerAPIList",
Description: "JUST FOR TEST",
URL: "https://a.b.c",
Adapter: "Clair",
Vendor: "Harbor",
Version: "0.1.0",
}}
suite.mockC.On("ListRegistrations", query).Return(ll, nil)
@ -198,9 +187,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPIUpdate() {
Name: "TestScannerAPIUpdate_before",
Description: "JUST FOR TEST",
URL: "https://a.b.c",
Adapter: "Clair",
Vendor: "Harbor",
Version: "0.1.0",
}
updated := &scanner.Registration{
@ -209,9 +195,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPIUpdate() {
Name: "TestScannerAPIUpdate",
Description: "JUST FOR TEST",
URL: "https://a.b.c",
Adapter: "Clair",
Vendor: "Harbor",
Version: "0.1.0",
}
suite.mockQuery(updated)
@ -240,9 +223,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPIDelete() {
Name: "TestScannerAPIDelete",
Description: "JUST FOR TEST",
URL: "https://a.b.c",
Adapter: "Clair",
Vendor: "Harbor",
Version: "0.1.0",
}
suite.mockC.On("DeleteRegistration", "uuid").Return(r, nil)
@ -299,9 +279,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPIProjectScanner() {
Name: "TestScannerAPIProjectScanner",
Description: "JUST FOR TEST",
URL: "https://a.b.c",
Adapter: "Clair",
Vendor: "Harbor",
Version: "0.1.0",
}
suite.mockC.On("GetRegistrationByProject", int64(1)).Return(r, nil)
@ -408,37 +385,3 @@ func (m *MockScannerAPIController) GetRegistrationByProject(projectID int64) (*s
return s.(*scanner.Registration), args.Error(1)
}
// Ping ...
func (m *MockScannerAPIController) Ping(registration *scanner.Registration) error {
args := m.Called(registration)
return args.Error(0)
}
// Scan ...
func (m *MockScannerAPIController) Scan(artifact *scan.Artifact) error {
args := m.Called(artifact)
return args.Error(0)
}
// GetReport ...
func (m *MockScannerAPIController) GetReport(artifact *scan.Artifact) ([]*dscan.Report, error) {
args := m.Called(artifact)
r := args.Get(0)
if r == nil {
return nil, args.Error(1)
}
return r.([]*dscan.Report), args.Error(1)
}
// GetScanLog ...
func (m *MockScannerAPIController) GetScanLog(digest string) ([]byte, error) {
args := m.Called(digest)
l := args.Get(0)
if l == nil {
return nil, args.Error(1)
}
return l.([]byte), args.Error(1)
}

2
src/core/router.go
View File

@ -195,7 +195,7 @@ func initRouters() {
// Add routes for plugin scanner management
scannerAPI := &api.ScannerAPI{}
beego.Router("/api/scanners", scannerAPI, "post:Create;get:List")
beego.Router("/api/scanners/:uid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault")
beego.Router("/api/scanners/:uuid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault")
// Add routes for project level scanner
beego.Router("/api/projects/:pid([0-9]+)/scanner", scannerAPI, "get:GetProjectScanner;put:SetProjectScanner")

58
src/pkg/scan/api/scan/base_controller.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
)
// basicController is default implementation of api.Controller interface
type basicController struct {
// Client for talking to scanner adapter
client v1.Client
}
// NewController news a scan API controller
func NewController() Controller {
return &basicController{}
}
// Scan ...
func (bc *basicController) Scan(artifact *v1.Artifact) error {
return nil
}
// GetReport ...
func (bc *basicController) GetReport(artifact *v1.Artifact) ([]*scan.Report, error) {
return nil, nil
}
// GetScanLog ...
func (bc *basicController) GetScanLog(digest string) ([]byte, error) {
return nil, nil
}
// Ping ...
func (bc *basicController) Ping(registration *scanner.Registration) error {
return nil
}
// HandleJobHooks ...
func (bc *basicController) HandleJobHooks(trackID int64, change *job.StatusChange) error {
return nil
}

78
src/pkg/scan/api/scan/controller.go Normal file
View File

@ -0,0 +1,78 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
)
// Controller provides the related operations for triggering scan.
// TODO: Here the artifact object is reused the v1 one which is sent to the adapter,
// it should be pointed to the general artifact object in future once it's ready.
type Controller interface {
// Ping pings Scanner Adapter to test EndpointURL and Authorization settings.
// The implementation is supposed to call the GetMetadata method on scanner.Client.
// Returns `nil` if connection succeeded, a non `nil` error otherwise.
//
// Arguments:
// registration *scanner.Registration : scanner registration to ping
//
// Returns:
// error : non nil error if any errors occurred
Ping(registration *scanner.Registration) error
// Scan the given artifact
//
// Arguments:
// artifact *v1.Artifact : artifact to be scanned
//
// Returns:
// error : non nil error if any errors occurred
Scan(artifact *v1.Artifact) error
// GetReport gets the reports for the given artifact identified by the digest
//
// Arguments:
// artifact *v1.Artifact : the scanned artifact
//
// Returns:
// []*scan.Report : scan results by different scanner vendors
// error : non nil error if any errors occurred
GetReport(artifact *v1.Artifact) ([]*scan.Report, error)
// Get the scan log for the specified artifact with the given digest
//
// Arguments:
// digest string : the digest of the artifact
//
// Returns:
// []byte : the log text stream
// error : non nil error if any errors occurred
GetScanLog(digest string) ([]byte, error)
// HandleJobHooks handle the hook events from the job service
// e.g : status change of the scan job or scan result
//
// Arguments:
// trackID int64 : ID for the result record
// change *job.StatusChange : change event from the job service
//
// Returns:
// error : non nil error if any errors occurred
HandleJobHooks(trackID int64, change *job.StatusChange) error
}

35
src/pkg/scan/scanner/api/registration.go → src/pkg/scan/api/scanner/base_controller.go
View File

@ -12,15 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package api
package scanner
import (
"github.com/goharbor/harbor/src/core/promgr/metamgr"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
rscanner "github.com/goharbor/harbor/src/pkg/scan/scanner"
"github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/scanner/scan"
"github.com/pkg/errors"
)
@ -45,9 +44,6 @@ type basicController struct {
manager rscanner.Manager
// for operating the project level configured scanner
proMetaMgr metamgr.ProjectMetadataManager
// controller for scan actions
c scan.Controller
// Client
}
// ListRegistrations ...
@ -58,9 +54,13 @@ func (bc *basicController) ListRegistrations(query *q.Query) ([]*scanner.Registr
// CreateRegistration ...
func (bc *basicController) CreateRegistration(registration *scanner.Registration) (string, error) {
// TODO: Get metadata from the adapter service first
l, err := bc.manager.List(nil)
// Check if there are any registrations already existing.
l, err := bc.manager.List(&q.Query{
PageSize: 1,
PageNumber: 1,
})
if err != nil {
return "", err
return "", errors.Wrap(err, "api controller: create registration")
}
if len(l) == 0 && !registration.IsDefault {
@ -102,7 +102,7 @@ func (bc *basicController) DeleteRegistration(registrationUUID string) (*scanner
}
if err := bc.manager.Delete(registrationUUID); err != nil {
return nil, errors.Wrap(err, "delete registration")
return nil, errors.Wrap(err, "api controller: delete registration")
}
return registration, nil
@ -127,7 +127,7 @@ func (bc *basicController) SetRegistrationByProject(projectID int64, registratio
// Scanner metadata existing?
m, err := bc.proMetaMgr.Get(projectID, proScannerMetaKey)
if err != nil {
return errors.Wrap(err, "set project scanner")
return errors.Wrap(err, "api controller: set project scanner")
}
// Update if exists
@ -136,14 +136,14 @@ func (bc *basicController) SetRegistrationByProject(projectID int64, registratio
if registrationID != m[proScannerMetaKey] {
m[proScannerMetaKey] = registrationID
if err := bc.proMetaMgr.Update(projectID, m); err != nil {
return errors.Wrap(err, "set project scanner")
return errors.Wrap(err, "api controller: set project scanner")
}
}
} else {
meta := make(map[string]string, 1)
meta[proScannerMetaKey] = registrationID
if err := bc.proMetaMgr.Add(projectID, meta); err != nil {
return errors.Wrap(err, "set project scanner")
return errors.Wrap(err, "api controller: set project scanner")
}
}
@ -159,21 +159,21 @@ func (bc *basicController) GetRegistrationByProject(projectID int64) (*scanner.R
// First, get it from the project metadata
m, err := bc.proMetaMgr.Get(projectID, proScannerMetaKey)
if err != nil {
return nil, errors.Wrap(err, "get project scanner")
return nil, errors.Wrap(err, "api controller: get project scanner")
}
if len(m) > 0 {
if registrationID, ok := m[proScannerMetaKey]; ok && len(registrationID) > 0 {
registration, err := bc.manager.Get(registrationID)
if err != nil {
return nil, errors.Wrap(err, "get project scanner")
return nil, errors.Wrap(err, "api controller: get project scanner")
}
if registration == nil {
// Not found
// Might be deleted by the admin, the project scanner ID reference should be cleared
if err := bc.proMetaMgr.Delete(projectID, proScannerMetaKey); err != nil {
return nil, errors.Wrap(err, "get project scanner")
return nil, errors.Wrap(err, "api controller: get project scanner")
}
} else {
return registration, nil
@ -187,8 +187,3 @@ func (bc *basicController) GetRegistrationByProject(projectID int64) (*scanner.R
// TODO: Check status by the client later
return registration, err
}
// Ping ...
func (bc *basicController) Ping(registration *scanner.Registration) error {
return nil
}

46
src/pkg/scan/scanner/api/controller.go → src/pkg/scan/api/scanner/controller.go
View File

@ -12,13 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package api
package scanner
import (
"github.com/goharbor/harbor/src/pkg/q"
dscan "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/scanner/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
)
// Controller provides the related operations of scanner for the upper API.
@ -114,44 +112,4 @@ type Controller interface {
// *scanner.Registration : the default scanner registration
// error : non nil error if any errors occurred
GetRegistrationByProject(projectID int64) (*scanner.Registration, error)
// Ping pings Scanner Adapter to test EndpointURL and Authorization settings.
// The implementation is supposed to call the GetMetadata method on scanner.Client.
// Returns `nil` if connection succeeded, a non `nil` error otherwise.
//
// Arguments:
// registration *scanner.Registration : scanner registration to ping
//
// Returns:
// error : non nil error if any errors occurred
Ping(registration *scanner.Registration) error
// Scan the given artifact
//
// Arguments:
// artifact *res.Artifact : artifact to be scanned
//
// Returns:
// error : non nil error if any errors occurred
Scan(artifact *scan.Artifact) error
// GetReport gets the reports for the given artifact identified by the digest
//
// Arguments:
// artifact *res.Artifact : the scanned artifact
//
// Returns:
// []*scan.Report : scan results by different scanner vendors
// error : non nil error if any errors occurred
GetReport(artifact *scan.Artifact) ([]*dscan.Report, error)
// Get the scan log for the specified artifact with the given digest
//
// Arguments:
// digest string : the digest of the artifact
//
// Returns:
// []byte : the log text stream
// error : non nil error if any errors occurred
GetScanLog(digest string) ([]byte, error)
}

7
src/pkg/scan/scanner/api/controller_test.go → src/pkg/scan/api/scanner/controller_test.go
View File

@ -12,14 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package api
package scanner
import (
"testing"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
@ -56,9 +56,6 @@ func (suite *ControllerTestSuite) SetupSuite() {
Name: "forUT",
Description: "sample registration",
URL: "https://sample.scanner.com",
Adapter: "Clair",
Version: "0.1.0",
Vendor: "Harbor",
}
}

14
src/pkg/scan/scanner/dao/scan/report.go → src/pkg/scan/dao/scan/model.go
View File

@ -16,15 +16,18 @@ package scan
import "time"
// Report of the scan
// Identified by the `digest` and `endpoint_id`
// Report of the scan.
// Identified by the `digest`, `registration_uuid` and `mime_type`.
type Report struct {
ID int64 `orm:"pk;auto;column(id)"`
UUID string `orm:"unique;column(uuid)"`
Digest string `orm:"column(digest)"`
ReregistrationID string `orm:"column(registration_id)"`
RegistrationUUID string `orm:"column(registration_uuid)"`
MimeType string `orm:"column(mime_type)"`
JobID string `orm:"column(job_id)"`
Status string `orm:"column(status)"`
StatusCode int `orm:"column(status_code)"`
StatusRevision int64 `orm:"column(status_rev)"`
Report string `orm:"column(report);type(json)"`
StartTime time.Time `orm:"column(start_time);auto_now_add;type(datetime)"`
EndTime time.Time `orm:"column(end_time);type(datetime)"`
@ -32,12 +35,13 @@ type Report struct {
// TableName for Report
func (r *Report) TableName() string {
return "scanner_report"
return "scan_report"
}
// TableUnique for Report
func (r *Report) TableUnique() [][]string {
return [][]string{
{"digest", "registration_id"},
{"uuid"},
{"digest", "registration_uuid", "mime_type"},
}
}

17
src/pkg/scan/scanner/dao/scanner/model.go → src/pkg/scan/dao/scanner/model.go
View File

@ -45,11 +45,6 @@ type Registration struct {
// Http connection settings
SkipCertVerify bool `orm:"column(skip_cert_verify);default(false)" json:"skip_certVerify"`
// Adapter settings
Adapter string `orm:"column(adapter);size(128)" json:"adapter"`
Vendor string `orm:"column(vendor);size(128)" json:"vendor"`
Version string `orm:"column(version);size(32)" json:"version"`
// Timestamps
CreateTime time.Time `orm:"column(create_time);auto_now_add;type(datetime)" json:"create_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now;type(datetime)" json:"update_time"`
@ -60,7 +55,7 @@ func (r *Registration) TableName() string {
return "scanner_registration"
}
// FromJSON parses json data
// FromJSON parses registration from json data
func (r *Registration) FromJSON(jsonData string) error {
if len(jsonData) == 0 {
return errors.New("empty json data to parse")
@ -69,7 +64,7 @@ func (r *Registration) FromJSON(jsonData string) error {
return json.Unmarshal([]byte(jsonData), r)
}
// ToJSON marshals endpoint to JSON data
// ToJSON marshals registration to JSON data
func (r *Registration) ToJSON() (string, error) {
data, err := json.Marshal(r)
if err != nil {
@ -79,7 +74,7 @@ func (r *Registration) ToJSON() (string, error) {
return string(data), nil
}
// Validate endpoint
// Validate registration
func (r *Registration) Validate(checkUUID bool) error {
if checkUUID && len(r.UUID) == 0 {
return errors.New("malformed endpoint")
@ -94,12 +89,6 @@ func (r *Registration) Validate(checkUUID bool) error {
return errors.Wrap(err, "scanner registration validate")
}
if len(r.Adapter) == 0 ||
len(r.Vendor) == 0 ||
len(r.Version) == 0 {
return errors.Errorf("missing adapter settings in registration %s:%s", r.Name, r.URL)
}
return nil
}

8
src/pkg/scan/scanner/dao/scanner/model_test.go → src/pkg/scan/dao/scanner/model_test.go
View File

@ -38,9 +38,6 @@ func (suite *ModelTestSuite) TestJSON() {
Name: "forUT",
Description: "sample registration",
URL: "https://sample.scanner.com",
Adapter: "Clair",
Version: "0.1.0",
Vendor: "Harbor",
}
json, err := r.ToJSON()
@ -77,11 +74,8 @@ func (suite *ModelTestSuite) TestValidate() {
r.URL = "http://a.b.c"
err = r.Validate(true)
require.Error(suite.T(), err)
require.NoError(suite.T(), err)
r.Adapter = "Clair"
r.Vendor = "Harbor"
r.Version = "0.1.0"
err = r.Validate(true)
require.NoError(suite.T(), err)
}

0
src/pkg/scan/scanner/dao/scanner/registration.go → src/pkg/scan/dao/scanner/registration.go
View File

7
src/pkg/scan/scanner/dao/scanner/registration_test.go → src/pkg/scan/dao/scanner/registration_test.go
View File

@ -50,9 +50,6 @@ func (suite *RegistrationDAOTestSuite) SetupTest() {
Name: "forUT",
Description: "sample registration",
URL: "https://sample.scanner.com",
Adapter: "Clair",
Version: "0.1.0",
Vendor: "Harbor",
}
_, err := AddRegistration(r)
@ -111,7 +108,7 @@ func (suite *RegistrationDAOTestSuite) TestList() {
// with query and found items
keywords := make(map[string]string)
keywords["adapter"] = "Clair"
keywords["description"] = "sample"
l, err = ListRegistrations(&q.Query{
PageSize: 5,
PageNumber: 1,
@ -121,7 +118,7 @@ func (suite *RegistrationDAOTestSuite) TestList() {
require.Equal(suite.T(), 1, len(l))
// With query and not found items
keywords["adapter"] = "Micro scanner"
keywords["description"] = "not_exist"
l, err = ListRegistrations(&q.Query{
Keywords: keywords,
})

337
src/pkg/scan/job.go Normal file
View File

@ -0,0 +1,337 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"bytes"
"encoding/json"
"fmt"
"reflect"
"sync"
"time"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/report"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/pkg/errors"
)
const (
// JobParamRegistration ...
JobParamRegistration = "registration"
// JobParameterRequest ...
JobParameterRequest = "scanRequest"
// JobParameterMimes ...
JobParameterMimes = "mimeTypes"
checkTimeout = 30 * time.Minute
firstCheckInterval = 2 * time.Second
)
// CheckInReport defines model for checking in the scan report with specified mime.
type CheckInReport struct {
Digest string `json:"digest"`
RegistrationUUID string `json:"registration_uuid"`
MimeType string `json:"mime_type"`
RawReport string `json:"raw_report"`
}
// FromJSON parse json to CheckInReport
func (cir *CheckInReport) FromJSON(jsonData string) error {
if len(jsonData) == 0 {
return errors.New("empty JSON data")
}
return json.Unmarshal([]byte(jsonData), cir)
}
// ToJSON marshal CheckInReport to JSON
func (cir *CheckInReport) ToJSON() (string, error) {
jsonData, err := json.Marshal(cir)
if err != nil {
return "", errors.Wrap(err, "To JSON: CheckInReport")
}
return string(jsonData), nil
}
// Job for running scan in the job service with async way
type Job struct{}
// MaxFails for defining the number of retries
func (j *Job) MaxFails() uint {
return 3
}
// ShouldRetry indicates if the job should be retried
func (j *Job) ShouldRetry() bool {
return true
}
// Validate the parameters of this job
func (j *Job) Validate(params job.Parameters) error {
if params == nil {
// Params are required
return errors.New("missing parameter of scan job")
}
if _, err := extractRegistration(params); err != nil {
return errors.Wrap(err, "job validate")
}
if _, err := extractScanReq(params); err != nil {
return errors.Wrap(err, "job validate")
}
if _, err := extractMimeTypes(params); err != nil {
return errors.Wrap(err, "job validate")
}
return nil
}
// Run the job
func (j *Job) Run(ctx job.Context, params job.Parameters) error {
// Get logger
myLogger := ctx.GetLogger()
// Ignore errors as they have been validated already
r, _ := extractRegistration(params)
req, _ := extractScanReq(params)
mimes, _ := extractMimeTypes(params)
// Print related infos to log
printJSONParameter(JobParamRegistration, params[JobParamRegistration].(string), myLogger)
printJSONParameter(JobParameterRequest, params[JobParameterRequest].(string), myLogger)
// Submit scan request to the scanner adapter
client, err := v1.DefaultClientPool.Get(r)
if err != nil {
return errors.Wrap(err, "run scan job")
}
resp, err := client.SubmitScan(req)
if err != nil {
return errors.Wrap(err, "run scan job")
}
// Exit signal
exit := make(chan bool, 1)
// Done signal
done := make(chan bool, 1)
// Collect errors
eChan := make(chan error, 1)
go func() {
defer func() {
// Done!
done <- true
}()
for {
select {
case e := <-eChan:
if err != nil {
err = errors.Wrap(e, err.Error())
} else {
err = e
}
case <-exit:
// Gracefully exit
return
case <-ctx.SystemContext().Done():
// Terminated by system
return
}
}
}()
// Concurrently retrieving report by different mime types
wg := &sync.WaitGroup{}
wg.Add(len(mimes))
for _, mt := range mimes {
go func(m string) {
defer wg.Done()
// Log info
myLogger.Infof("Get report for mime type: %s", m)
// Loop check if the report is ready
tm := time.NewTimer(firstCheckInterval)
defer tm.Stop()
for {
select {
case t := <-tm.C:
myLogger.Debugf("check scan report for mime %s at %s", m, t.Format("2006/01/02 15:04:05"))
rawReport, err := client.GetScanReport(resp.ID, m)
if err != nil {
// Not ready yet
if notReadyErr, ok := err.(*v1.ReportNotReadyError); ok {
// Reset to the new check interval
tm.Reset(time.Duration(notReadyErr.RetryAfter) * time.Second)
myLogger.Infof("Report with mime type %s is not ready yet, retry after %d seconds", m, notReadyErr.RetryAfter)
continue
}
eChan <- errors.Wrap(err, fmt.Sprintf("check scan report with mime type %s", m))
return
}
// Make sure the data is aligned with the v1 spec.
if _, err = report.ResolveData(m, []byte(rawReport)); err != nil {
eChan <- errors.Wrap(err, "scan job: resolve report data")
return
}
// Check in
cir := &CheckInReport{
Digest: req.Artifact.Digest,
RegistrationUUID: r.UUID,
MimeType: m,
RawReport: rawReport,
}
var (
jsonData string
er error
)
if jsonData, er = cir.ToJSON(); er == nil {
if er = ctx.Checkin(jsonData); er == nil {
// Done!
myLogger.Infof("Report with mime type %s is checked in", m)
return
}
}
// Send error and exit
eChan <- errors.Wrap(er, fmt.Sprintf("check in scan report for mime type %s", m))
return
case <-ctx.SystemContext().Done():
// Terminated by system
return
case <-time.After(checkTimeout):
eChan <- errors.New("check scan report timeout")
return
}
}
}(mt)
}
// Wait for all the retrieving routines are completed
wg.Wait()
// Stop error collection goroutine
exit <- true
// done!
<-done
// Log error to the job log
if err != nil {
myLogger.Error(err)
}
return err
}
func printJSONParameter(parameter string, v string, logger logger.Interface) {
logger.Infof("%s:\n", parameter)
printPrettyJSON([]byte(v), logger)
}
func printPrettyJSON(in []byte, logger logger.Interface) {
var out bytes.Buffer
if err := json.Indent(&out, in, "", " "); err != nil {
logger.Errorf("Print pretty JSON error: %s", err)
return
}
logger.Infof("%s\n", out.String())
}
func extractScanReq(params job.Parameters) (*v1.ScanRequest, error) {
v, ok := params[JobParameterRequest]
if !ok {
return nil, errors.Errorf("missing job parameter '%s'", JobParameterRequest)
}
jsonData, ok := v.(string)
if !ok {
return nil, errors.Errorf(
"malformed job parameter '%s', expecting string but got %s",
JobParameterRequest,
reflect.TypeOf(v).String(),
)
}
req := &v1.ScanRequest{}
if err := req.FromJSON(jsonData); err != nil {
return nil, err
}
if err := req.Validate(); err != nil {
return nil, err
}
return req, nil
}
func extractRegistration(params job.Parameters) (*scanner.Registration, error) {
v, ok := params[JobParamRegistration]
if !ok {
return nil, errors.Errorf("missing job parameter '%s'", JobParamRegistration)
}
jsonData, ok := v.(string)
if !ok {
return nil, errors.Errorf(
"malformed job parameter '%s', expecting string but got %s",
JobParamRegistration,
reflect.TypeOf(v).String(),
)
}
r := &scanner.Registration{}
if err := r.FromJSON(jsonData); err != nil {
return nil, err
}
if err := r.Validate(true); err != nil {
return nil, err
}
return r, nil
}
func extractMimeTypes(params job.Parameters) ([]string, error) {
v, ok := params[JobParameterMimes]
if !ok {
return nil, errors.Errorf("missing job parameter '%s'", JobParameterMimes)
}
l, ok := v.([]string)
if !ok {
return nil, errors.Errorf(
"malformed job parameter '%s', expecting string but got %s",
JobParameterMimes,
reflect.TypeOf(v).String(),
)
}
return l, nil
}

306
src/pkg/scan/job_test.go Normal file
View File

@ -0,0 +1,306 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
// JobTestSuite is a test suite to test the scan job.
type JobTestSuite struct {
suite.Suite
defaultClientPool v1.ClientPool
mcp *MockClientPool
}
// TestJob is the entry of JobTestSuite.
func TestJob(t *testing.T) {
suite.Run(t, &JobTestSuite{})
}
// SetupSuite sets up test env for JobTestSuite.
func (suite *JobTestSuite) SetupSuite() {
mcp := &MockClientPool{}
suite.defaultClientPool = v1.DefaultClientPool
v1.DefaultClientPool = mcp
suite.mcp = mcp
}
// TeraDownSuite clears test env for TeraDownSuite.
func (suite *JobTestSuite) TeraDownSuite() {
v1.DefaultClientPool = suite.defaultClientPool
}
// TestJob tests the scan job
func (suite *JobTestSuite) TestJob() {
ctx := &MockJobContext{}
lg := &MockJobLogger{}
ctx.On("GetLogger").Return(lg)
r := &scanner.Registration{
ID: 0,
UUID: "uuid",
Name: "TestJob",
URL: "https://clair.com:8080",
}
rData, err := r.ToJSON()
require.NoError(suite.T(), err)
sr := &v1.ScanRequest{
Registry: &v1.Registry{
URL: "http://localhost:5000",
Authorization: "the_token",
},
Artifact: &v1.Artifact{
Repository: "library/test_job",
Digest: "sha256:data",
MimeType: v1.MimeTypeDockerArtifact,
},
}
sData, err := sr.ToJSON()
require.NoError(suite.T(), err)
mimeTypes := []string{v1.MimeTypeNativeReport}
jp := make(job.Parameters)
jp[JobParamRegistration] = rData
jp[JobParameterRequest] = sData
jp[JobParameterMimes] = mimeTypes
mc := &MockClient{}
sre := &v1.ScanResponse{
ID: "scan_id",
}
mc.On("SubmitScan", sr).Return(sre, nil)
rp := vuln.Report{
GeneratedAt: time.Now().UTC().String(),
Scanner: &v1.Scanner{
Name: "Clair",
Vendor: "Harbor",
Version: "0.1.0",
},
Severity: vuln.High,
Vulnerabilities: []*vuln.VulnerabilityItem{
{
ID: "2019-0980-0909",
Package: "dpkg",
Version: "0.9.1",
FixVersion: "0.9.2",
Severity: vuln.High,
Description: "mock one",
Links: []string{"https://vuln.com"},
},
},
}
jRep, err := json.Marshal(rp)
require.NoError(suite.T(), err)
mc.On("GetScanReport", "scan_id", v1.MimeTypeNativeReport).Return(string(jRep), nil)
suite.mcp.On("Get", r).Return(mc, nil)
crp := &CheckInReport{
Digest: sr.Artifact.Digest,
RegistrationUUID: r.UUID,
MimeType: v1.MimeTypeNativeReport,
RawReport: string(jRep),
}
jsonData, err := crp.ToJSON()
require.NoError(suite.T(), err)
ctx.On("Checkin", string(jsonData)).Return(nil)
j := &Job{}
err = j.Run(ctx, jp)
require.NoError(suite.T(), err)
}
// MockJobContext mocks job context interface.
// TODO: Maybe moved to a separate `mock` pkg for sharing in future.
type MockJobContext struct {
mock.Mock
}
// Build ...
func (mjc *MockJobContext) Build(tracker job.Tracker) (job.Context, error) {
args := mjc.Called(tracker)
c := args.Get(0)
if c != nil {
return c.(job.Context), nil
}
return nil, args.Error(1)
}
// Get ...
func (mjc *MockJobContext) Get(prop string) (interface{}, bool) {
args := mjc.Called(prop)
return args.Get(0), args.Bool(1)
}
// SystemContext ...
func (mjc *MockJobContext) SystemContext() context.Context {
return context.TODO()
}
// Checkin ...
func (mjc *MockJobContext) Checkin(status string) error {
args := mjc.Called(status)
return args.Error(0)
}
// OPCommand ...
func (mjc *MockJobContext) OPCommand() (job.OPCommand, bool) {
args := mjc.Called()
return (job.OPCommand)(args.String(0)), args.Bool(1)
}
// GetLogger ...
func (mjc *MockJobContext) GetLogger() logger.Interface {
return &MockJobLogger{}
}
// Tracker ...
func (mjc *MockJobContext) Tracker() job.Tracker {
args := mjc.Called()
if t := args.Get(0); t != nil {
return t.(job.Tracker)
}
return nil
}
// MockJobLogger mocks the job logger interface.
// TODO: Maybe moved to a separate `mock` pkg for sharing in future.
type MockJobLogger struct {
mock.Mock
}
// Debug ...
func (mjl *MockJobLogger) Debug(v ...interface{}) {
logger.Debug(v...)
}
// Debugf ...
func (mjl *MockJobLogger) Debugf(format string, v ...interface{}) {
logger.Debugf(format, v...)
}
// Info ...
func (mjl *MockJobLogger) Info(v ...interface{}) {
logger.Info(v...)
}
// Infof ...
func (mjl *MockJobLogger) Infof(format string, v ...interface{}) {
logger.Infof(format, v...)
}
// Warning ...
func (mjl *MockJobLogger) Warning(v ...interface{}) {
logger.Warning(v...)
}
// Warningf ...
func (mjl *MockJobLogger) Warningf(format string, v ...interface{}) {
logger.Warningf(format, v...)
}
// Error ...
func (mjl *MockJobLogger) Error(v ...interface{}) {
logger.Error(v...)
}
// Errorf ...
func (mjl *MockJobLogger) Errorf(format string, v ...interface{}) {
logger.Errorf(format, v...)
}
// Fatal ...
func (mjl *MockJobLogger) Fatal(v ...interface{}) {
logger.Fatal(v...)
}
// Fatalf ...
func (mjl *MockJobLogger) Fatalf(format string, v ...interface{}) {
logger.Fatalf(format, v...)
}
// MockClientPool mocks the client pool
type MockClientPool struct {
mock.Mock
}
// Get v1 client
func (mcp *MockClientPool) Get(r *scanner.Registration) (v1.Client, error) {
args := mcp.Called(r)
c := args.Get(0)
if c != nil {
return c.(v1.Client), nil
}
return nil, args.Error(1)
}
// MockClient mocks the v1 client
type MockClient struct {
mock.Mock
}
// GetMetadata ...
func (mc *MockClient) GetMetadata() (*v1.ScannerAdapterMetadata, error) {
args := mc.Called()
s := args.Get(0)
if s != nil {
return s.(*v1.ScannerAdapterMetadata), nil
}
return nil, args.Error(1)
}
// SubmitScan ...
func (mc *MockClient) SubmitScan(req *v1.ScanRequest) (*v1.ScanResponse, error) {
args := mc.Called(req)
sr := args.Get(0)
if sr != nil {
return sr.(*v1.ScanResponse), nil
}
return nil, args.Error(1)
}
// GetScanReport ...
func (mc *MockClient) GetScanReport(scanRequestID, reportMIMEType string) (string, error) {
args := mc.Called(scanRequestID, reportMIMEType)
return args.String(0), args.Error(1)
}

78
src/pkg/scan/report/supported_mime_test.go Normal file
View File

@ -0,0 +1,78 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package report
import (
"encoding/json"
"testing"
"time"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
// SupportedMimesSuite is a suite to test SupportedMimes.
type SupportedMimesSuite struct {
suite.Suite
mockData []byte
}
// TestSupportedMimesSuite is the entry of SupportedMimesSuite.
func TestSupportedMimesSuite(t *testing.T) {
suite.Run(t, new(SupportedMimesSuite))
}
// SetupSuite prepares the test suite env.
func (suite *SupportedMimesSuite) SetupSuite() {
rp := vuln.Report{
GeneratedAt: time.Now().UTC().String(),
Scanner: &v1.Scanner{
Name: "Clair",
Vendor: "Harbor",
Version: "0.1.0",
},
Severity: vuln.High,
Vulnerabilities: []*vuln.VulnerabilityItem{
{
ID: "2019-0980-0909",
Package: "dpkg",
Version: "0.9.1",
FixVersion: "0.9.2",
Severity: vuln.High,
Description: "mock one",
Links: []string{"https://vuln.com"},
},
},
}
jsonData, err := json.Marshal(rp)
require.NoError(suite.T(), err)
suite.mockData = jsonData
}
// TestResolveData tests the ResolveData.
func (suite *SupportedMimesSuite) TestResolveData() {
obj, err := ResolveData(v1.MimeTypeNativeReport, suite.mockData)
require.NoError(suite.T(), err)
require.Condition(suite.T(), func() (success bool) {
rp, ok := obj.(*vuln.Report)
success = ok && rp != nil && rp.Severity == vuln.High
return
})
}

56
src/pkg/scan/report/supported_mimes.go Normal file
View File

@ -0,0 +1,56 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package report
import (
"encoding/json"
"reflect"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
"github.com/pkg/errors"
)
// SupportedMimes indicates what mime types are supported to render at UI end.
var SupportedMimes = map[string]interface{}{
// The native report type
v1.MimeTypeNativeReport: (*vuln.Report)(nil),
}
// ResolveData is a helper func to parse the JSON data with the given mime type.
func ResolveData(mime string, jsonData []byte) (interface{}, error) {
if len(jsonData) == 0 {
return nil, errors.New("empty JSON data")
}
t, ok := SupportedMimes[mime]
if !ok {
return nil, errors.Errorf("report with mime type %s is not supported", mime)
}
ty := reflect.TypeOf(t)
if ty.Kind() == reflect.Ptr {
ty = ty.Elem()
}
// New one
rp := reflect.New(ty).Elem().Addr().Interface()
if err := json.Unmarshal(jsonData, rp); err != nil {
return nil, err
}
return rp, nil
}

45
src/pkg/scan/rest/auth/api_key_auth.go Normal file
View File

@ -0,0 +1,45 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"net/http"
"github.com/pkg/errors"
)
// apiKeyAuthorizer authorize by adding a header `X-ScannerAdapter-API-Key` with value "credential"
type apiKeyAuthorizer struct {
typeID string
accessCred string
}
// Authorize the requests
func (aa *apiKeyAuthorizer) Authorize(req *http.Request) error {
if req != nil && len(aa.accessCred) > 0 {
req.Header.Add(aa.typeID, aa.accessCred)
return nil
}
return errors.Errorf("%s: %s", aa.typeID, "missing data to authorize request")
}
// NewAPIKeyAuthorizer news a apiKeyAuthorizer
func NewAPIKeyAuthorizer(accessCred string) Authorizer {
return &apiKeyAuthorizer{
typeID: APIKey,
accessCred: accessCred,
}
}

54
src/pkg/scan/rest/auth/auth.go Normal file
View File

@ -0,0 +1,54 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"net/http"
"strings"
"github.com/pkg/errors"
)
const (
authorization = "Authorization"
// Basic ...
Basic = "Basic"
// Bearer ...
Bearer = "Bearer"
// APIKey ...
APIKey = "X-ScannerAdapter-API-Key"
)
// Authorizer defines operation for authorizing the requests
type Authorizer interface {
Authorize(req *http.Request) error
}
// GetAuthorizer is a factory method for getting an authorizer based on the given auth type
func GetAuthorizer(auth, cred string) (Authorizer, error) {
switch strings.TrimSpace(auth) {
// No authorizer required
case "":
return NewNoAuth(), nil
case Basic:
return NewBasicAuth(cred), nil
case Bearer:
return NewBearerAuth(cred), nil
case APIKey:
return NewAPIKeyAuthorizer(cred), nil
default:
return nil, errors.Errorf("auth type %s is not supported", auth)
}
}

48
src/pkg/scan/rest/auth/basic_auth.go Normal file
View File

@ -0,0 +1,48 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"encoding/base64"
"fmt"
"net/http"
"github.com/pkg/errors"
)
// basicAuthorizer authorizes the request by adding `Authorization Basic base64(credential)` header
type basicAuthorizer struct {
typeID string
accessCred string
}
// Authorize requests
func (ba *basicAuthorizer) Authorize(req *http.Request) error {
if len(ba.accessCred) == 0 {
return errors.Errorf("%s:%s", ba.typeID, "missing access credential")
}
if req != nil && len(ba.accessCred) > 0 {
data := base64.StdEncoding.EncodeToString([]byte(ba.accessCred))
req.Header.Add(authorization, fmt.Sprintf("%s %s", ba.typeID, data))
}
return errors.Errorf("%s: %s", ba.typeID, "missing data to authorize request")
}
// NewBasicAuth basic authorizer
func NewBasicAuth(accessCred string) Authorizer {
return &basicAuthorizer{Basic, accessCred}
}

40
src/pkg/scan/rest/auth/bearer_auth.go Normal file
View File

@ -0,0 +1,40 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"fmt"
"net/http"
)
// bearerAuthorizer authorizes the request by adding `Authorization Bearer credential` header
type bearerAuthorizer struct {
typeID string
accessCred string
}
// Authorize requests
func (ba *bearerAuthorizer) Authorize(req *http.Request) error {
if req != nil && len(ba.accessCred) > 0 {
req.Header.Add(authorization, fmt.Sprintf("%s %s", ba.typeID, ba.accessCred))
}
return nil
}
// NewBearerAuth create bearer authorizer
func NewBearerAuth(accessCred string) Authorizer {
return &bearerAuthorizer{Bearer, accessCred}
}

24
src/pkg/scan/scanner/api/scan.go → src/pkg/scan/rest/auth/no_auth.go
View File

@ -12,24 +12,22 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package api
package auth
import (
dscan "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/scanner/scan"
"net/http"
)
// Scan ...
func (bc *basicController) Scan(artifact *scan.Artifact) error {
// noAuth is created to handle the no authorization case which is acceptable
type noAuth struct{}
// Authorize the incoming request
func (na *noAuth) Authorize(req *http.Request) error {
// Do nothing
return nil
}
// GetReport ...
func (bc *basicController) GetReport(artifact *scan.Artifact) ([]*dscan.Report, error) {
return nil, nil
}
// GetScanLog ...
func (bc *basicController) GetScanLog(digest string) ([]byte, error) {
return nil, nil
// NewNoAuth creates a noAuth authorizer
func NewNoAuth() Authorizer {
return &noAuth{}
}

278
src/pkg/scan/rest/v1/client.go Normal file
View File

@ -0,0 +1,278 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"bytes"
"crypto/tls"
"encoding/json"
"io/ioutil"
"net"
"net/http"
"strconv"
"time"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/rest/auth"
"github.com/pkg/errors"
)
const (
// defaultRefreshInterval is the default interval with seconds of refreshing report
defaultRefreshInterval = 5
// refreshAfterHeader provides the refresh interval value
refreshAfterHeader = "Refresh-After"
)
// Client defines the methods to access the adapter services that
// implement the REST API specs
type Client interface {
// GetMetadata gets the metadata of the given scanner
//
// Returns:
// *ScannerAdapterMetadata : metadata of the given scanner
// error : non nil error if any errors occurred
GetMetadata() (*ScannerAdapterMetadata, error)
// SubmitScan initiates a scanning of the given artifact.
// Returns `nil` if the request was accepted, a non `nil` error otherwise.
//
// Arguments:
// req *ScanRequest : request including the registry and artifact data
//
// Returns:
// *ScanResponse : response with UUID for tracking the scan results
// error : non nil error if any errors occurred
SubmitScan(req *ScanRequest) (*ScanResponse, error)
// GetScanReport gets the scan result for the corresponding ScanRequest identifier.
// Note that this is a blocking method which either returns a non `nil` scan report or error.
// A caller is supposed to cast the returned interface{} to a structure that corresponds
// to the specified MIME type.
//
// Arguments:
// scanRequestID string : the ID of the scan submitted before
// reportMIMEType string : the report mime type
// Returns:
// string : the scan report of the given artifact
// error : non nil error if any errors occurred
GetScanReport(scanRequestID, reportMIMEType string) (string, error)
}
// basicClient is default implementation of the Client interface
type basicClient struct {
httpClient *http.Client
spec *Spec
authorizer auth.Authorizer
}
// NewClient news a basic client
func NewClient(r *scanner.Registration) (Client, error) {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: r.SkipCertVerify,
},
}
authorizer, err := auth.GetAuthorizer(r.Auth, r.AccessCredential)
if err != nil {
return nil, errors.Wrap(err, "new v1 client")
}
return &basicClient{
httpClient: &http.Client{
Transport: transport,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
},
spec: NewSpec(r.URL),
authorizer: authorizer,
}, nil
}
// GetMetadata ...
func (c *basicClient) GetMetadata() (*ScannerAdapterMetadata, error) {
def := c.spec.Metadata()
request, err := http.NewRequest(http.MethodGet, def.URL, nil)
if err != nil {
return nil, errors.Wrap(err, "v1 client: get metadata")
}
// Resolve header
def.Resolver(request)
// Send request
respData, err := c.send(request, generalResponseHandler(http.StatusOK))
if err != nil {
return nil, errors.Wrap(err, "v1 client: get metadata")
}
// Unmarshal data
meta := &ScannerAdapterMetadata{}
if err := json.Unmarshal(respData, meta); err != nil {
return nil, errors.Wrap(err, "v1 client: get metadata")
}
return meta, nil
}
// SubmitScan ...
func (c *basicClient) SubmitScan(req *ScanRequest) (*ScanResponse, error) {
if req == nil {
return nil, errors.New("nil request")
}
data, err := json.Marshal(req)
if err != nil {
return nil, errors.Wrap(err, "v1 client: submit scan")
}
def := c.spec.SubmitScan()
request, err := http.NewRequest(http.MethodPost, def.URL, bytes.NewReader(data))
if err != nil {
return nil, errors.Wrap(err, "v1 client: submit scan")
}
respData, err := c.send(request, generalResponseHandler(http.StatusCreated))
if err != nil {
return nil, errors.Wrap(err, "v1 client: submit scan")
}
resp := &ScanResponse{}
if err := json.Unmarshal(respData, resp); err != nil {
return nil, errors.Wrap(err, "v1 client: submit scan")
}
return resp, nil
}
// GetScanReport ...
func (c *basicClient) GetScanReport(scanRequestID, reportMIMEType string) (string, error) {
if len(scanRequestID) == 0 {
return "", errors.New("empty scan request ID")
}
if len(reportMIMEType) == 0 {
return "", errors.New("missing report mime type")
}
def := c.spec.GetScanReport(scanRequestID, reportMIMEType)
req, err := http.NewRequest(http.MethodGet, def.URL, nil)
if err != nil {
return "", errors.Wrap(err, "v1 client: get scan report")
}
respData, err := c.send(req, reportResponseHandler())
if err != nil {
// This error should not be wrapped
return "", err
}
return string(respData), nil
}
func (c *basicClient) send(req *http.Request, h responseHandler) ([]byte, error) {
if c.authorizer != nil {
if err := c.authorizer.Authorize(req); err != nil {
return nil, errors.Wrap(err, "authorization")
}
}
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, err
}
defer func() {
if err := resp.Body.Close(); err != nil {
// Just logged
logger.Errorf("close response body error: %s", err)
}
}()
return h(resp.StatusCode, resp)
}
// responseHandlerFunc is a handler func template for handling the http response data,
// especially the error part.
type responseHandler func(code int, resp *http.Response) ([]byte, error)
// generalResponseHandler create a general response handler to cover the common cases.
func generalResponseHandler(expectedCode int) responseHandler {
return func(code int, resp *http.Response) ([]byte, error) {
return generalRespHandlerFunc(expectedCode, code, resp)
}
}
// reportResponseHandler creates response handler for get report special case.
func reportResponseHandler() responseHandler {
return func(code int, resp *http.Response) ([]byte, error) {
if code == http.StatusFound {
// Set default
retryAfter := defaultRefreshInterval // seconds
// Read `retry after` info from header
v := resp.Header.Get(refreshAfterHeader)
if len(v) > 0 {
if i, err := strconv.ParseInt(v, 10, 8); err == nil {
retryAfter = int(i)
} else {
// log error
logger.Errorf("Parse `%s` error: %s", refreshAfterHeader, err)
}
}
return nil, &ReportNotReadyError{RetryAfter: retryAfter}
}
return generalRespHandlerFunc(http.StatusOK, code, resp)
}
}
// generalRespHandlerFunc is a handler to cover the general cases
func generalRespHandlerFunc(expectedCode, code int, resp *http.Response) ([]byte, error) {
buf, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if code != expectedCode {
if len(buf) > 0 {
// Try to read error response
eResp := &ErrorResponse{
Err: &Error{},
}
if err := json.Unmarshal(buf, eResp); err == nil {
return nil, eResp
}
}
return nil, errors.Errorf("unexpected status code: %d, response: %s", code, string(buf))
}
return buf, nil
}

128
src/pkg/scan/rest/v1/client_pool.go Normal file
View File

@ -0,0 +1,128 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"encoding/base64"
"fmt"
"sync"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/pkg/errors"
)
// DefaultClientPool is a default client pool.
var DefaultClientPool = NewClientPool()
// ClientPool defines operations for the client pool which provides v1 client cache.
type ClientPool interface {
// Get a v1 client interface for the specified registration.
//
// Arguments:
// r *scanner.Registration : registration for client connecting to
//
// Returns:
// Client : v1 client
// error : non nil error if any errors occurred
Get(r *scanner.Registration) (Client, error)
}
// basicClientPool is default implementation of client pool interface.
type basicClientPool struct {
pool *sync.Map
}
// NewClientPool news a basic client pool.
func NewClientPool() ClientPool {
return &basicClientPool{
pool: &sync.Map{},
}
}
// Get client for the specified registration.
// So far, there will not be too many scanner registrations. An then
// no need to do client instance clear work.
// If one day, we have to clear unactivated client instances in the pool,
// add the following func after the first time initializing the client.
// pool item represents the client with a timestamp of last accessed.
//
// type poolItem struct {
// c Client
// timestamp time.Time
// }
//
// func (bcp *basicClientPool) deadCheck(key string, item *poolItem) {
// // Run in a separate goroutine
// go func() {
// // As we do not have a global context, let's watch the system signal to
// // exit the goroutine correctly.
// sig := make(chan os.Signal, 1)
// signal.Notify(sig, os.Interrupt, syscall.SIGTERM, os.Kill)
//
// tk := time.NewTicker(bcp.config.DeadCheckInterval)
// defer tk.Stop()
//
// for {
// select {
// case t := <-tk.C:
// if item.timestamp.Add(bcp.config.ExpireTime).Before(t.UTC()) {
// // Expired
// bcp.pool.Delete(key)
// return
// }
// case <-sig:
// // Terminated by system
// return
// }
// }
// }()
// }
func (bcp *basicClientPool) Get(r *scanner.Registration) (Client, error) {
if r == nil {
return nil, errors.New("nil scanner registration")
}
if err := r.Validate(true); err != nil {
return nil, errors.Wrap(err, "client pool: get")
}
k := key(r)
c, ok := bcp.pool.Load(k)
if !ok {
nc, err := NewClient(r)
if err != nil {
return nil, errors.Wrap(err, "client pool: get")
}
// Cache it
bcp.pool.Store(k, nc)
c = nc
}
return c.(Client), nil
}
func key(r *scanner.Registration) string {
raw := fmt.Sprintf("%s:%s:%s:%s:%v",
r.UUID,
r.URL,
r.Auth,
r.AccessCredential,
r.SkipCertVerify,
)
return base64.StdEncoding.EncodeToString([]byte(raw))
}

69
src/pkg/scan/rest/v1/client_pool_test.go Normal file
View File

@ -0,0 +1,69 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"fmt"
"testing"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/rest/auth"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
// ClientPoolTestSuite is a test suite to test the client pool.
type ClientPoolTestSuite struct {
suite.Suite
pool ClientPool
}
// TestClientPool is the entry of ClientPoolTestSuite.
func TestClientPool(t *testing.T) {
suite.Run(t, new(ClientPoolTestSuite))
}
// SetupSuite sets up test suite env.
func (suite *ClientPoolTestSuite) SetupSuite() {
suite.pool = NewClientPool()
}
// TestClientPoolGet tests the get method of client pool.
func (suite *ClientPoolTestSuite) TestClientPoolGet() {
r := &scanner.Registration{
ID: 1,
Name: "TestClientPoolGet",
UUID: "uuid",
URL: "http://a.b.c",
Auth: auth.Basic,
AccessCredential: "u:p",
SkipCertVerify: false,
}
client1, err := suite.pool.Get(r)
require.NoError(suite.T(), err)
require.NotNil(suite.T(), client1)
p1 := fmt.Sprintf("%p", client1.(*basicClient))
client2, err := suite.pool.Get(r)
require.NoError(suite.T(), err)
require.NotNil(suite.T(), client2)
p2 := fmt.Sprintf("%p", client2.(*basicClient))
assert.Equal(suite.T(), p1, p2)
}

197
src/pkg/scan/rest/v1/client_test.go Normal file
View File

@ -0,0 +1,197 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
// ClientTestSuite tests the v1 client
type ClientTestSuite struct {
suite.Suite
testServer *httptest.Server
client Client
}
// TestClient is the entry of ClientTestSuite
func TestClient(t *testing.T) {
suite.Run(t, new(ClientTestSuite))
}
// SetupSuite prepares the test suite env
func (suite *ClientTestSuite) SetupSuite() {
suite.testServer = httptest.NewServer(&mockHandler{})
r := &scanner.Registration{
ID: 1000,
UUID: "uuid",
Name: "TestClient",
URL: suite.testServer.URL,
SkipCertVerify: true,
}
c, err := NewClient(r)
require.NoError(suite.T(), err)
require.NotNil(suite.T(), c)
suite.client = c
}
// TestClientMetadata tests the metadata of the client
func (suite *ClientTestSuite) TestClientMetadata() {
m, err := suite.client.GetMetadata()
require.NoError(suite.T(), err)
require.NotNil(suite.T(), m)
assert.Equal(suite.T(), m.Scanner.Name, "Clair")
}
// TestClientSubmitScan tests the scan submission of client
func (suite *ClientTestSuite) TestClientSubmitScan() {
res, err := suite.client.SubmitScan(&ScanRequest{})
require.NoError(suite.T(), err)
require.NotNil(suite.T(), res)
assert.Equal(suite.T(), res.ID, "123456789")
}
// TestClientGetScanReportError tests getting report failed
func (suite *ClientTestSuite) TestClientGetScanReportError() {
_, err := suite.client.GetScanReport("id1", MimeTypeNativeReport)
require.Error(suite.T(), err)
assert.Condition(suite.T(), func() (success bool) {
success = strings.Index(err.Error(), "error") != -1
return
})
}
// TestClientGetScanReport tests getting report
func (suite *ClientTestSuite) TestClientGetScanReport() {
res, err := suite.client.GetScanReport("id2", MimeTypeNativeReport)
require.NoError(suite.T(), err)
require.NotEmpty(suite.T(), res)
}
// TestClientGetScanReportNotReady tests the case that the report is not ready
func (suite *ClientTestSuite) TestClientGetScanReportNotReady() {
_, err := suite.client.GetScanReport("id3", MimeTypeNativeReport)
require.Error(suite.T(), err)
require.Condition(suite.T(), func() (success bool) {
_, success = err.(*ReportNotReadyError)
return
})
assert.Equal(suite.T(), 10, err.(*ReportNotReadyError).RetryAfter)
}
// TearDownSuite clears the test suite env
func (suite *ClientTestSuite) TearDownSuite() {
suite.testServer.Close()
}
type mockHandler struct{}
// ServeHTTP ...
func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
switch r.RequestURI {
case "/metadata":
if r.Method != http.MethodGet {
w.WriteHeader(http.StatusForbidden)
return
}
m := &ScannerAdapterMetadata{
Scanner: &Scanner{
Name: "Clair",
Vendor: "Harbor",
Version: "0.1.0",
},
Capabilities: &ScannerCapability{
ConsumesMimeTypes: []string{
MimeTypeOCIArtifact,
MimeTypeDockerArtifact,
},
ProducesMimeTypes: []string{
MimeTypeNativeReport,
MimeTypeRawReport,
},
},
Properties: ScannerProperties{
"extra": "testing",
},
}
data, _ := json.Marshal(m)
w.WriteHeader(http.StatusOK)
_, _ = w.Write(data)
break
case "/scan":
if r.Method != http.MethodPost {
w.WriteHeader(http.StatusForbidden)
return
}
res := &ScanResponse{}
res.ID = "123456789"
data, _ := json.Marshal(res)
w.WriteHeader(http.StatusCreated)
_, _ = w.Write(data)
break
case "/scan/id1/report":
if r.Method != http.MethodGet {
w.WriteHeader(http.StatusForbidden)
return
}
e := &ErrorResponse{
&Error{
Message: "error",
},
}
data, _ := json.Marshal(e)
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write(data)
break
case "/scan/id2/report":
if r.Method != http.MethodGet {
w.WriteHeader(http.StatusForbidden)
return
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("{}"))
break
case "/scan/id3/report":
if r.Method != http.MethodGet {
w.WriteHeader(http.StatusForbidden)
return
}
w.Header().Add(refreshAfterHeader, fmt.Sprintf("%d", 10))
w.Header().Add("Location", "/scan/id3/report")
w.WriteHeader(http.StatusFound)
break
}
}

173
src/pkg/scan/rest/v1/models.go Normal file
View File

@ -0,0 +1,173 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"encoding/json"
"fmt"
"github.com/pkg/errors"
)
// Scanner represents metadata of a Scanner Adapter which allow Harbor to lookup a scanner capable of
// scanning a given Artifact stored in its registry and making sure that it can interpret a
// returned result.
type Scanner struct {
// The name of the scanner.
Name string `json:"name"`
// The name of the scanner's provider.
Vendor string `json:"vendor"`
// The version of the scanner.
Version string `json:"version"`
}
// ScannerCapability consists of the set of recognized artifact MIME types and the set of scanner
// report MIME types. For example, a scanner capable of analyzing Docker images and producing
// a vulnerabilities report recognizable by Harbor web console might be represented with the
// following capability:
// - consumes MIME types:
// -- application/vnd.oci.image.manifest.v1+json
// -- application/vnd.docker.distribution.manifest.v2+json
// - produces MIME types
// -- application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0
// -- application/vnd.scanner.adapter.vuln.report.raw
type ScannerCapability struct {
// The set of MIME types of the artifacts supported by the scanner to produce the reports
// specified in the "produces_mime_types". A given mime type should only be present in one
// capability item.
ConsumesMimeTypes []string `json:"consumes_mime_types"`
// The set of MIME types of reports generated by the scanner for the consumes_mime_types of
// the same capability record.
ProducesMimeTypes []string `json:"produces_mime_types"`
}
// ScannerProperties is a set of custom properties that can further describe capabilities of a given scanner.
type ScannerProperties map[string]string
// ScannerAdapterMetadata represents metadata of a Scanner Adapter which allows Harbor to lookup
// a scanner capable of scanning a given Artifact stored in its registry and making sure that it
// can interpret a returned result.
type ScannerAdapterMetadata struct {
Scanner *Scanner `json:"scanner"`
Capabilities *ScannerCapability `json:"capabilities"`
Properties ScannerProperties `json:"properties"`
}
// Artifact represents an artifact stored in Registry.
type Artifact struct {
// The full name of a Harbor repository containing the artifact, including the namespace.
// For example, `library/oracle/nosql`.
Repository string `json:"repository"`
// The artifact's digest, consisting of an algorithm and hex portion.
// For example, `sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b`,
// represents sha256 based digest.
Digest string `json:"digest"`
// The mime type of the scanned artifact
MimeType string `json:"mime_type"`
}
// Registry represents Registry connection settings.
type Registry struct {
// A base URL of the Docker Registry v2 API exposed by Harbor.
URL string `json:"url"`
// An optional value of the HTTP Authorization header sent with each request to the Docker Registry v2 API.
// For example, `Bearer: JWTTOKENGOESHERE`.
Authorization string `json:"authorization"`
}
// ScanRequest represents a structure that is sent to a Scanner Adapter to initiate artifact scanning.
// Conducts all the details required to pull the artifact from a Harbor registry.
type ScanRequest struct {
// Connection settings for the Docker Registry v2 API exposed by Harbor.
Registry *Registry `json:"registry"`
// Artifact to be scanned.
Artifact *Artifact `json:"artifact"`
}
// FromJSON parses ScanRequest from json data
func (s *ScanRequest) FromJSON(jsonData string) error {
if len(jsonData) == 0 {
return errors.New("empty json data to parse")
}
return json.Unmarshal([]byte(jsonData), s)
}
// ToJSON marshals ScanRequest to JSON data
func (s *ScanRequest) ToJSON() (string, error) {
data, err := json.Marshal(s)
if err != nil {
return "", err
}
return string(data), nil
}
// Validate ScanRequest
func (s *ScanRequest) Validate() error {
if s.Registry == nil ||
len(s.Registry.URL) == 0 ||
len(s.Registry.Authorization) == 0 {
return errors.New("scan request: invalid registry")
}
if s.Artifact == nil ||
len(s.Artifact.Digest) == 0 ||
len(s.Artifact.Repository) == 0 ||
len(s.Artifact.MimeType) == 0 {
return errors.New("scan request: invalid artifact")
}
return nil
}
// ScanResponse represents the response returned by the scanner adapter after scan request successfully
// submitted.
type ScanResponse struct {
// e.g: 3fa85f64-5717-4562-b3fc-2c963f66afa6
ID string `json:"id"`
}
// ErrorResponse contains error message when requests are not correctly handled.
type ErrorResponse struct {
// Error object
Err *Error `json:"error"`
}
// Error message
type Error struct {
// Message of the error
Message string `json:"message"`
}
// Error for ErrorResponse
func (er *ErrorResponse) Error() string {
if er.Err != nil {
return er.Err.Message
}
return "nil error"
}
// ReportNotReadyError is an error to indicate the scan report is not ready
type ReportNotReadyError struct {
// Seconds for next retry with seconds
RetryAfter int
}
// Error for ReportNotReadyError
func (rnr *ReportNotReadyError) Error() string {
return fmt.Sprintf("report is not ready yet, retry after %d", rnr.RetryAfter)
}

109
src/pkg/scan/rest/v1/spec.go Normal file
View File

@ -0,0 +1,109 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
import (
"fmt"
"net/http"
"strings"
)
const (
// HTTPAcceptHeader represents the HTTP accept header
HTTPAcceptHeader = "Accept"
// HTTPContentType represents the HTTP content-type header
HTTPContentType = "Content-Type"
// MimeTypeOCIArtifact defines the mime type for OCI artifact
MimeTypeOCIArtifact = "application/vnd.oci.image.manifest.v1+json"
// MimeTypeDockerArtifact defines the mime type for docker artifact
MimeTypeDockerArtifact = "application/vnd.docker.distribution.manifest.v2+json"
// MimeTypeNativeReport defines the mime type for native report
MimeTypeNativeReport = "application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"
// MimeTypeRawReport defines the mime type for raw report
MimeTypeRawReport = "application/vnd.scanner.adapter.vuln.report.raw"
// MimeTypeAdapterMeta defines the mime type for adapter metadata
MimeTypeAdapterMeta = "application/vnd.scanner.adapter.metadata+json; version=1.0"
// MimeTypeScanRequest defines the mime type for scan request
MimeTypeScanRequest = "application/vnd.scanner.adapter.scan.request+json; version=1.0"
// MimeTypeScanResponse defines the mime type for scan response
MimeTypeScanResponse = "application/vnd.scanner.adapter.scan.response+json; version=1.0"
)
// RequestResolver is a function template to modify the API request, e.g: add headers
type RequestResolver func(req *http.Request)
// Definition for API
type Definition struct {
// URL of the API
URL string
// Resolver fro the request
Resolver RequestResolver
}
// Spec of the API
// Contains URL and possible headers.
type Spec struct {
baseRoute string
}
// NewSpec news V1 spec
func NewSpec(base string) *Spec {
baseRoute := "http://localhost"
if len(base) > 0 {
if strings.HasSuffix(base, "/") {
baseRoute = base[:len(base)-1]
} else {
baseRoute = base
}
}
return &Spec{
baseRoute: baseRoute,
}
}
// Metadata API
func (s *Spec) Metadata() Definition {
return Definition{
URL: fmt.Sprintf("%s%s", s.baseRoute, "/metadata"),
Resolver: func(req *http.Request) {
req.Header.Add(HTTPAcceptHeader, MimeTypeAdapterMeta)
},
}
}
// SubmitScan API
func (s *Spec) SubmitScan() Definition {
return Definition{
URL: fmt.Sprintf("%s%s", s.baseRoute, "/scan"),
Resolver: func(req *http.Request) {
req.Header.Add(HTTPContentType, MimeTypeScanRequest)
req.Header.Add(HTTPAcceptHeader, MimeTypeScanResponse)
},
}
}
// GetScanReport API
func (s *Spec) GetScanReport(scanReqID string, mimeType string) Definition {
path := fmt.Sprintf("/scan/%s/report", scanReqID)
return Definition{
URL: fmt.Sprintf("%s%s", s.baseRoute, path),
Resolver: func(req *http.Request) {
req.Header.Add(HTTPAcceptHeader, mimeType)
},
}
}

8
src/pkg/scan/scanner/manager.go
View File

@ -16,7 +16,7 @@ package scanner
import (
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/google/uuid"
"github.com/pkg/errors"
)
@ -59,7 +59,7 @@ func New() Manager {
// Create ...
func (bm *basicManager) Create(registration *scanner.Registration) (string, error) {
if registration == nil {
return "", errors.New("nil endpoint to create")
return "", errors.New("nil registration to create")
}
// Inject new UUID
@ -92,11 +92,11 @@ func (bm *basicManager) Get(registrationUUID string) (*scanner.Registration, err
// Update ...
func (bm *basicManager) Update(registration *scanner.Registration) error {
if registration == nil {
return errors.New("nil endpoint to update")
return errors.New("nil registration to update")
}
if err := registration.Validate(true); err != nil {
return errors.Wrap(err, "update endpoint")
return errors.Wrap(err, "update registration")
}
return scanner.UpdateRegistration(registration)

5
src/pkg/scan/scanner/manager_test.go
View File

@ -19,7 +19,7 @@ import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
@ -48,9 +48,6 @@ func (suite *BasicManagerTestSuite) SetupSuite() {
Name: "forUT",
Description: "sample registration",
URL: "https://sample.scanner.com",
Adapter: "Clair",
Version: "0.1.0",
Vendor: "Harbor",
}
uid, err := suite.mgr.Create(r)

48
src/pkg/scan/scanner/scan/controller.go
View File

@ -1,48 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import "github.com/goharbor/harbor/src/pkg/scan/scanner/dao/scan"
// Options object for the scan action
type Options struct{}
// Option for scan action
type Option interface {
// Apply option to the passing in options
Apply(options *Options) error
}
// Controller defines operations for scan controlling
type Controller interface {
// Scan the given artifact
//
// Arguments:
// artifact *res.Artifact : artifact to be scanned
//
// Returns:
// error : non nil error if any errors occurred
Scan(artifact *Artifact, options ...Option) error
// GetReport gets the reports for the given artifact identified by the digest
//
// Arguments:
// artifact *res.Artifact : the scanned artifact
//
// Returns:
// []*scan.Report : scan results by different scanner vendors
// error : non nil error if any errors occurred
GetReport(artifact *Artifact) ([]*scan.Report, error)
}

46
src/pkg/scan/scanner/scan/models.go
View File

@ -1,46 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
// Artifact represents an artifact stored in Registry.
type Artifact struct {
// The full name of a Harbor repository containing the artifact, including the namespace.
// For example, `library/oracle/nosql`.
Repository string
// The artifact's digest, consisting of an algorithm and hex portion.
// For example, `sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b`,
// represents sha256 based digest.
Digest string
// The mime type of the scanned artifact
MimeType string
}
// Registry represents Registry connection settings.
type Registry struct {
// A base URL of the Docker Registry v2 API exposed by Harbor.
URL string
// An optional value of the HTTP Authorization header sent with each request to the Docker Registry v2 API.
// For example, `Bearer: JWTTOKENGOESHERE`.
Authorization string
}
// Request represents a structure that is sent to a Scanner Adapter to initiate artifact scanning.
// Conducts all the details required to pull the artifact from a Harbor registry.
type Request struct {
// Connection settings for the Docker Registry v2 API exposed by Harbor.
Registry *Registry
// Artifact to be scanned.
Artifact *Artifact
}

58
src/pkg/scan/vuln/report.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package vuln
import (
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
)
// Report model for vulnerability scan
type Report struct {
// Time of generating this report
GeneratedAt string `json:"generated_at"`
// Scanner of generating this report
Scanner *v1.Scanner `json:"scanner"`
// A standard scale for measuring the severity of a vulnerability.
Severity Severity `json:"severity"`
// Vulnerability list
Vulnerabilities []*VulnerabilityItem `json:"vulnerabilities"`
}
// VulnerabilityItem represents one found vulnerability
type VulnerabilityItem struct {
// The unique identifier of the vulnerability.
// e.g: CVE-2017-8283
ID string `json:"id"`
// An operating system or software dependency package containing the vulnerability.
// e.g: dpkg
Package string `json:"package"`
// The version of the package containing the vulnerability.
// e.g: 1.17.27
Version string `json:"version"`
// The version of the package containing the fix if available.
// e.g: 1.18.0
FixVersion string `json:"fix_version"`
// A standard scale for measuring the severity of a vulnerability.
Severity Severity `json:"severity"`
// example: dpkg-source in dpkg 1.3.0 through 1.18.23 is able to use a non-GNU patch program
// and does not offer a protection mechanism for blank-indented diff hunks, which allows remote
// attackers to conduct directory traversal attacks via a crafted Debian source package, as
// demonstrated by using of dpkg-source on NetBSD.
Description string `json:"description"`
// The list of link to the upstream database with the full description of the vulnerability.
// Format: URI
// e.g: List [ "https://security-tracker.debian.org/tracker/CVE-2017-8283" ]
Links []string
}

39
src/pkg/scan/vuln/severity.go Normal file
View File

@ -0,0 +1,39 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package vuln
const (
// Unknown - either a security problem that has not been assigned to a priority yet or
// a priority that the scanner did not recognize.
Unknown Severity = "Unknown"
// Low - a security problem, but is hard to exploit due to environment, requires a
// user-assisted attack, a small install base, or does very little damage.
Low Severity = "Low"
// Negligible - technically a security problem, but is only theoretical in nature, requires
// a very special situation, has almost no install base, or does no real damage.
Negligible Severity = "Negligible"
// Medium - a real security problem, and is exploitable for many people. Includes network
// daemon denial of service attacks, cross-site scripting, and gaining user privileges.
Medium Severity = "Medium"
// High - a real problem, exploitable for many people in a default installation. Includes
// serious remote denial of service, local root privilege escalations, or data loss.
High Severity = "High"
// Critical - a world-burning problem, exploitable for nearly all people in a default installation.
// Includes remote root privilege escalations, or massive data loss.
Critical Severity = "Critical"
)
// Severity is a standard scale for measuring the severity of a vulnerability.
type Severity string