Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-22 10:15:35 +01:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/goharbor/harbor into robot-invisiable

Browse Source
This commit is contained in:
wang yan 2019-10-14 14:35:45 +08:00
commit 25f638a989
60 changed files with 2815 additions and 503 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
docs/swagger.yaml
View File

@ -959,13 +959,13 @@ paths:
description: User ID does not exist.
'500':
description: Unexpected internal errors.
'/users/{user_id}/gen_cli_secret':
post:
summary: Generate new CLI secret for a user.
'/users/{user_id}/cli_secret':
put:
summary: Set CLI secret for a user.
description: |
This endpoint let user generate a new CLI secret for himself. This API only works when auth mode is set to 'OIDC'.
Once this API returns with successful status, the old secret will be invalid, as there will be only one CLI secret
for a user. The new secret will be returned in the response.
for a user.
parameters:
- name: user_id
in: path
@ -973,19 +973,23 @@ paths:
format: int
required: true
description: User ID
tags:
- Products
responses:
'200':
description: The secret is successfully generated.
- name: input_secret
in: body
description: JSON object that includes the new secret
required: true
schema:
type: object
properties:
secret:
type: string
description: The new secret
tags:
- Products
responses:
'200':
description: The secret is successfully updated
'400':
description: Invalid user ID. Or user is not onboarded via OIDC authentication.
description: Invalid user ID. Or user is not onboarded via OIDC authentication. Or the secret does not meet the standard.
'401':
description: User need to log in first.
'403':

2
docs/user_guide.md
View File

@ -906,7 +906,7 @@ For example, you have following tags, listed according to their push time, and a
You configure a retention policy to retain the two latest tags that match `harbor-*`, so that `harbor-rc` and `harbor-latest` are deleted. However, since all tags refer to the same SHA digest, this policy would also delete the tags `harbor-1.8` and `harbor-release`, so all tags are retained.
### Combining Rules on a Respository
### Combining Rules on a Repository
You can define up to 15 rules per project. You can apply multiple rules to a repository or set of repositories. When you apply multiple rules to a repository, they are applied with `OR` logic rather than with `AND` logic. In this way, there is no prioritization of application of the rules on a given repository. Rules run concurrently in the background, and the resulting sets from each rule are combined at the end of the run.

34
make/migrations/postgresql/0012_1.10.0_schema.up.sql
View File

@ -1,34 +0,0 @@
/*Table for keeping the plug scanner registration*/
CREATE TABLE scanner_registration
(
id SERIAL PRIMARY KEY NOT NULL,
uuid VARCHAR(64) UNIQUE NOT NULL,
url VARCHAR(256) UNIQUE NOT NULL,
name VARCHAR(128) UNIQUE NOT NULL,
description VARCHAR(1024) NULL,
auth VARCHAR(16) NOT NULL,
access_cred VARCHAR(512) NULL,
disabled BOOLEAN NOT NULL DEFAULT FALSE,
is_default BOOLEAN NOT NULL DEFAULT FALSE,
skip_cert_verify BOOLEAN NOT NULL DEFAULT FALSE,
create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
/*Table for keeping the scan report. The report details are stored as JSON*/
CREATE TABLE scan_report
(
id SERIAL PRIMARY KEY NOT NULL,
uuid VARCHAR(64) UNIQUE NOT NULL,
digest VARCHAR(256) NOT NULL,
registration_uuid VARCHAR(64) NOT NULL,
mime_type VARCHAR(256) NOT NULL,
job_id VARCHAR(32),
status VARCHAR(16) NOT NULL,
status_code INTEGER DEFAULT 0,
status_rev BIGINT DEFAULT 0,
report JSON,
start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
end_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE(digest, registration_uuid, mime_type)
)

36
make/migrations/postgresql/0015_1.10.0_schema.up.sql
View File

@ -1,3 +1,39 @@
/*Table for keeping the plug scanner registration*/
CREATE TABLE scanner_registration
(
id SERIAL PRIMARY KEY NOT NULL,
uuid VARCHAR(64) UNIQUE NOT NULL,
url VARCHAR(256) UNIQUE NOT NULL,
name VARCHAR(128) UNIQUE NOT NULL,
description VARCHAR(1024) NULL,
auth VARCHAR(16) NOT NULL,
access_cred VARCHAR(512) NULL,
disabled BOOLEAN NOT NULL DEFAULT FALSE,
is_default BOOLEAN NOT NULL DEFAULT FALSE,
skip_cert_verify BOOLEAN NOT NULL DEFAULT FALSE,
create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
/*Table for keeping the scan report. The report details are stored as JSON*/
CREATE TABLE scan_report
(
id SERIAL PRIMARY KEY NOT NULL,
uuid VARCHAR(64) UNIQUE NOT NULL,
digest VARCHAR(256) NOT NULL,
registration_uuid VARCHAR(64) NOT NULL,
mime_type VARCHAR(256) NOT NULL,
job_id VARCHAR(64),
track_id VARCHAR(64),
status VARCHAR(1024) NOT NULL,
status_code INTEGER DEFAULT 0,
status_rev BIGINT DEFAULT 0,
report JSON,
start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
end_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE(digest, registration_uuid, mime_type)
);
/** Add table for immutable tag **/
CREATE TABLE immutable_tag_rule
(

10
src/common/models/repo.go
View File

@ -54,11 +54,11 @@ type RepositoryQuery struct {
// TagResp holds the information of one image tag
type TagResp struct {
TagDetail
Signature *model.Target `json:"signature"`
ScanOverview *ImgScanOverview `json:"scan_overview,omitempty"`
Labels []*Label `json:"labels"`
PushTime time.Time `json:"push_time"`
PullTime time.Time `json:"pull_time"`
Signature *model.Target `json:"signature"`
ScanOverview map[string]interface{} `json:"scan_overview,omitempty"`
Labels []*Label `json:"labels"`
PushTime time.Time `json:"push_time"`
PullTime time.Time `json:"pull_time"`
}
// TagDetail ...

5
src/common/rbac/const.go
View File

@ -54,9 +54,10 @@ const (
ResourceRepositoryTag = Resource("repository-tag")
ResourceRepositoryTagLabel = Resource("repository-tag-label")
ResourceRepositoryTagManifest = Resource("repository-tag-manifest")
ResourceRepositoryTagScanJob = Resource("repository-tag-scan-job")
ResourceRepositoryTagVulnerability = Resource("repository-tag-vulnerability")
ResourceRepositoryTagScanJob = Resource("repository-tag-scan-job") // TODO: remove
ResourceRepositoryTagVulnerability = Resource("repository-tag-vulnerability") // TODO: remove
ResourceRobot = Resource("robot")
ResourceNotificationPolicy = Resource("notification-policy")
ResourceScan = Resource("scan")
ResourceSelf = Resource("") // subresource for self
)

3
src/common/rbac/project/util.go
View File

@ -162,6 +162,9 @@ var (
{Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionDelete},
{Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList},
{Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionRead},
{Resource: rbac.ResourceScan, Action: rbac.ActionCreate},
{Resource: rbac.ResourceScan, Action: rbac.ActionRead},
}
)

9
src/common/rbac/project/visitor_role.go
View File

@ -119,6 +119,9 @@ var (
{Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionDelete},
{Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList},
{Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionRead},
{Resource: rbac.ResourceScan, Action: rbac.ActionCreate},
{Resource: rbac.ResourceScan, Action: rbac.ActionRead},
},
"master": {
@ -201,6 +204,9 @@ var (
{Resource: rbac.ResourceRobot, Action: rbac.ActionList},
{Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList},
{Resource: rbac.ResourceScan, Action: rbac.ActionCreate},
{Resource: rbac.ResourceScan, Action: rbac.ActionRead},
},
"developer": {
@ -251,6 +257,9 @@ var (
{Resource: rbac.ResourceRobot, Action: rbac.ActionRead},
{Resource: rbac.ResourceRobot, Action: rbac.ActionList},
{Resource: rbac.ResourceScan, Action: rbac.ActionCreate},
{Resource: rbac.ResourceScan, Action: rbac.ActionRead},
},
"guest": {

45
src/common/utils/registry/auth/apikey.go Normal file
View File

@ -0,0 +1,45 @@
package auth
import (
"fmt"
"net/http"
"github.com/goharbor/harbor/src/common/http/modifier"
)
type apiKeyType = string
const (
// APIKeyInHeader sets auth content in header
APIKeyInHeader apiKeyType = "header"
// APIKeyInQuery sets auth content in url query
APIKeyInQuery apiKeyType = "query"
)
type apiKeyAuthorizer struct {
key, value, in apiKeyType
}
// NewAPIKeyAuthorizer returns a apikey authorizer
func NewAPIKeyAuthorizer(key, value, in apiKeyType) modifier.Modifier {
return &apiKeyAuthorizer{
key: key,
value: value,
in: in,
}
}
// Modify implements modifier.Modifier
func (a *apiKeyAuthorizer) Modify(r *http.Request) error {
switch a.in {
case APIKeyInHeader:
r.Header.Set(a.key, a.value)
return nil
case APIKeyInQuery:
query := r.URL.Query()
query.Add(a.key, a.value)
r.URL.RawQuery = query.Encode()
return nil
}
return fmt.Errorf("set api key in %s is invalid", a.in)
}

50
src/common/utils/registry/auth/apikey_test.go Normal file
View File

@ -0,0 +1,50 @@
package auth
import (
"net/http"
"testing"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/stretchr/testify/assert"
)
func TestAPIKeyAuthorizer(t *testing.T) {
type suite struct {
key string
value string
in string
}
var (
s suite
authorizer modifier.Modifier
request *http.Request
err error
)
// set in header
s = suite{key: "Authorization", value: "Basic abc", in: "header"}
authorizer = NewAPIKeyAuthorizer(s.key, s.value, s.in)
request, err = http.NewRequest(http.MethodGet, "http://example.com", nil)
assert.Nil(t, err)
err = authorizer.Modify(request)
assert.Nil(t, err)
assert.Equal(t, s.value, request.Header.Get(s.key))
// set in query
s = suite{key: "private_token", value: "abc", in: "query"}
authorizer = NewAPIKeyAuthorizer(s.key, s.value, s.in)
request, err = http.NewRequest(http.MethodGet, "http://example.com", nil)
assert.Nil(t, err)
err = authorizer.Modify(request)
assert.Nil(t, err)
assert.Equal(t, s.value, request.URL.Query().Get(s.key))
// set in invalid location
s = suite{key: "", value: "", in: "invalid"}
authorizer = NewAPIKeyAuthorizer(s.key, s.value, s.in)
request, err = http.NewRequest(http.MethodGet, "http://example.com", nil)
assert.Nil(t, err)
err = authorizer.Modify(request)
assert.NotNil(t, err)
}

11
src/core/api/harborapi_test.go
View File

@ -211,8 +211,17 @@ func init() {
scannerAPI := &ScannerAPI{}
beego.Router("/api/scanners", scannerAPI, "post:Create;get:List")
beego.Router("/api/scanners/:uuid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault")
beego.Router("/api/scanners/:uuid/metadata", scannerAPI, "get:Metadata")
beego.Router("/api/scanners/ping", scannerAPI, "post:Ping")
// Add routes for project level scanner
beego.Router("/api/projects/:pid([0-9]+)/scanner", scannerAPI, "get:GetProjectScanner;put:SetProjectScanner")
proScannerAPI := &ProjectScannerAPI{}
beego.Router("/api/projects/:pid([0-9]+)/scanner", proScannerAPI, "get:GetProjectScanner;put:SetProjectScanner")
// Add routes for scan
scanAPI := &ScanAPI{}
beego.Router("/api/repositories/*/tags/:tag/scan", scanAPI, "post:Scan;get:Report")
beego.Router("/api/repositories/*/tags/:tag/scan/:uuid/log", scanAPI, "get:Log")
// syncRegistry
if err := SyncRegistry(config.GlobalProjectMgr); err != nil {

112
src/core/api/pro_scanner.go Normal file
View File

@ -0,0 +1,112 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/pkg/scan/api/scanner"
"github.com/pkg/errors"
)
// ProjectScannerAPI provides rest API for managing the project level scanner(s).
type ProjectScannerAPI struct {
// The base controller to provide common utilities
BaseController
// Scanner controller for operating scanner registrations.
c scanner.Controller
// ID of the project
pid int64
}
// Prepare sth. for the subsequent actions
func (sa *ProjectScannerAPI) Prepare() {
// Call super prepare method
sa.BaseController.Prepare()
// Check access permissions
if !sa.RequireAuthenticated() {
return
}
// Get ID of the project
pid, err := sa.GetInt64FromPath(":pid")
if err != nil {
sa.SendBadRequestError(errors.Wrap(err, "project scanner API"))
return
}
// Check if the project exists
exists, err := sa.ProjectMgr.Exists(pid)
if err != nil {
sa.SendInternalServerError(errors.Wrap(err, "project scanner API"))
return
}
if !exists {
sa.SendNotFoundError(errors.Errorf("project with id %d", sa.pid))
return
}
sa.pid = pid
sa.c = scanner.DefaultController
}
// GetProjectScanner gets the project level scanner
func (sa *ProjectScannerAPI) GetProjectScanner() {
// Check access permissions
if !sa.RequireProjectAccess(sa.pid, rbac.ActionRead, rbac.ResourceConfiguration) {
return
}
r, err := sa.c.GetRegistrationByProject(sa.pid)
if err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scanner API: get project scanners"))
return
}
if r != nil {
sa.Data["json"] = r
} else {
sa.Data["json"] = make(map[string]interface{})
}
sa.ServeJSON()
}
// SetProjectScanner sets the project level scanner
func (sa *ProjectScannerAPI) SetProjectScanner() {
// Check access permissions
if !sa.RequireProjectAccess(sa.pid, rbac.ActionUpdate, rbac.ResourceConfiguration) {
return
}
body := make(map[string]string)
if err := sa.DecodeJSONReq(&body); err != nil {
sa.SendBadRequestError(errors.Wrap(err, "scanner API: set project scanners"))
return
}
uuid, ok := body["uuid"]
if !ok || len(uuid) == 0 {
sa.SendBadRequestError(errors.New("missing scanner uuid when setting project scanner"))
return
}
if err := sa.c.SetRegistrationByProject(sa.pid, uuid); err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scanner API: set project scanners"))
return
}
}

95
src/core/api/pro_scanner_test.go Normal file
View File

@ -0,0 +1,95 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"fmt"
"net/http"
"testing"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner"
"github.com/stretchr/testify/suite"
)
// ProScannerAPITestSuite is test suite for testing the project scanner API
type ProScannerAPITestSuite struct {
suite.Suite
originC sc.Controller
mockC *MockScannerAPIController
}
// TestProScannerAPI is the entry of ProScannerAPITestSuite
func TestProScannerAPI(t *testing.T) {
suite.Run(t, new(ProScannerAPITestSuite))
}
// SetupSuite prepares testing env
func (suite *ProScannerAPITestSuite) SetupTest() {
suite.originC = sc.DefaultController
m := &MockScannerAPIController{}
sc.DefaultController = m
suite.mockC = m
}
// TearDownTest clears test case env
func (suite *ProScannerAPITestSuite) TearDownTest() {
// Restore
sc.DefaultController = suite.originC
}
// TestScannerAPIProjectScanner tests the API of getting/setting project level scanner
func (suite *ProScannerAPITestSuite) TestScannerAPIProjectScanner() {
suite.mockC.On("SetRegistrationByProject", int64(1), "uuid").Return(nil)
// Set
body := make(map[string]interface{}, 1)
body["uuid"] = "uuid"
runCodeCheckingCases(suite.T(), &codeCheckingCase{
request: &testingRequest{
url: fmt.Sprintf("/api/projects/%d/scanner", 1),
method: http.MethodPut,
credential: projAdmin,
bodyJSON: body,
},
code: http.StatusOK,
})
r := &scanner.Registration{
ID: 1004,
UUID: "uuid",
Name: "TestScannerAPIProjectScanner",
Description: "JUST FOR TEST",
URL: "https://a.b.c",
}
suite.mockC.On("GetRegistrationByProject", int64(1)).Return(r, nil)
// Get
rr := &scanner.Registration{}
err := handleAndParse(&testingRequest{
url: fmt.Sprintf("/api/projects/%d/scanner", 1),
method: http.MethodGet,
credential: projAdmin,
}, rr)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), r.Name, rr.Name)
assert.Equal(suite.T(), r.UUID, rr.UUID)
}

157
src/core/api/repository.go
View File

@ -25,6 +25,11 @@ import (
"strings"
"time"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common"
@ -40,7 +45,6 @@ import (
"github.com/goharbor/harbor/src/core/config"
notifierEvt "github.com/goharbor/harbor/src/core/notifier/event"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/pkg/scan"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/model"
@ -397,6 +401,13 @@ func (ra *RepositoryAPI) GetTag() {
return
}
project, err := ra.ProjectMgr.Get(projectName)
if err != nil {
ra.ParseAndHandleError(fmt.Sprintf("failed to get the project %s",
projectName), err)
return
}
client, err := coreutils.NewRepositoryClientForUI(ra.SecurityCtx.GetUsername(), repository)
if err != nil {
ra.SendInternalServerError(fmt.Errorf("failed to initialize the client for %s: %v",
@ -414,7 +425,7 @@ func (ra *RepositoryAPI) GetTag() {
return
}
result := assembleTagsInParallel(client, repository, []string{tag},
result := assembleTagsInParallel(client, project.ProjectID, repository, []string{tag},
ra.SecurityCtx.GetUsername())
ra.Data["json"] = result[0]
ra.ServeJSON()
@ -523,14 +534,14 @@ func (ra *RepositoryAPI) GetTags() {
}
projectName, _ := utils.ParseRepository(repoName)
exist, err := ra.ProjectMgr.Exists(projectName)
project, err := ra.ProjectMgr.Get(projectName)
if err != nil {
ra.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s",
ra.ParseAndHandleError(fmt.Sprintf("failed to get the project %s",
projectName), err)
return
}
if !exist {
if project == nil {
ra.SendNotFoundError(fmt.Errorf("project %s not found", projectName))
return
}
@ -587,8 +598,13 @@ func (ra *RepositoryAPI) GetTags() {
return
}
ra.Data["json"] = assembleTagsInParallel(client, repoName, tags,
ra.SecurityCtx.GetUsername())
ra.Data["json"] = assembleTagsInParallel(
client,
project.ProjectID,
repoName,
tags,
ra.SecurityCtx.GetUsername(),
)
ra.ServeJSON()
}
@ -607,7 +623,7 @@ func simpleTags(tags []string) []*models.TagResp {
// get config, signature and scan overview and assemble them into one
// struct for each tag in tags
func assembleTagsInParallel(client *registry.Repository, repository string,
func assembleTagsInParallel(client *registry.Repository, projectID int64, repository string,
tags []string, username string) []*models.TagResp {
var err error
signatures := map[string][]notarymodel.Target{}
@ -621,8 +637,15 @@ func assembleTagsInParallel(client *registry.Repository, repository string,
c := make(chan *models.TagResp)
for _, tag := range tags {
go assembleTag(c, client, repository, tag, config.WithClair(),
config.WithNotary(), signatures)
go assembleTag(
c,
client,
projectID,
repository,
tag,
config.WithNotary(),
signatures,
)
}
result := []*models.TagResp{}
var item *models.TagResp
@ -636,8 +659,8 @@ func assembleTagsInParallel(client *registry.Repository, repository string,
return result
}
func assembleTag(c chan *models.TagResp, client *registry.Repository,
repository, tag string, clairEnabled, notaryEnabled bool,
func assembleTag(c chan *models.TagResp, client *registry.Repository, projectID int64,
repository, tag string, notaryEnabled bool,
signatures map[string][]notarymodel.Target) {
item := &models.TagResp{}
// labels
@ -659,8 +682,9 @@ func assembleTag(c chan *models.TagResp, client *registry.Repository,
}
// scan overview
if clairEnabled {
item.ScanOverview = getScanOverview(item.Digest, item.Name)
so := getSummary(projectID, repository, item.Digest)
if len(so) > 0 {
item.ScanOverview = so
}
// signature, compare both digest and tag
@ -968,73 +992,6 @@ func (ra *RepositoryAPI) GetSignatures() {
ra.ServeJSON()
}
// ScanImage handles request POST /api/repository/$repository/tags/$tag/scan to trigger image scan manually.
func (ra *RepositoryAPI) ScanImage() {
if !config.WithClair() {
log.Warningf("Harbor is not deployed with Clair, scan is disabled.")
ra.SendInternalServerError(errors.New("harbor is not deployed with Clair, scan is disabled"))
return
}
repoName := ra.GetString(":splat")
tag := ra.GetString(":tag")
projectName, _ := utils.ParseRepository(repoName)
exist, err := ra.ProjectMgr.Exists(projectName)
if err != nil {
ra.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s",
projectName), err)
return
}
if !exist {
ra.SendNotFoundError(fmt.Errorf("project %s not found", projectName))
return
}
if !ra.SecurityCtx.IsAuthenticated() {
ra.SendUnAuthorizedError(errors.New("Unauthorized"))
return
}
if !ra.RequireProjectAccess(projectName, rbac.ActionCreate, rbac.ResourceRepositoryTagScanJob) {
return
}
err = coreutils.TriggerImageScan(repoName, tag)
if err != nil {
log.Errorf("Error while calling job service to trigger image scan: %v", err)
ra.SendInternalServerError(errors.New("Failed to scan image, please check log for details"))
return
}
}
// VulnerabilityDetails fetch vulnerability info from clair, transform to Harbor's format and return to client.
func (ra *RepositoryAPI) VulnerabilityDetails() {
if !config.WithClair() {
log.Warningf("Harbor is not deployed with Clair, it's not impossible to get vulnerability details.")
ra.SendInternalServerError(errors.New("harbor is not deployed with Clair, it's not impossible to get vulnerability details"))
return
}
repository := ra.GetString(":splat")
tag := ra.GetString(":tag")
exist, digest, err := ra.checkExistence(repository, tag)
if err != nil {
ra.SendInternalServerError(fmt.Errorf("failed to check the existence of resource, error: %v", err))
return
}
if !exist {
ra.SendNotFoundError(fmt.Errorf("resource: %s:%s not found", repository, tag))
return
}
projectName, _ := utils.ParseRepository(repository)
if !ra.RequireProjectAccess(projectName, rbac.ActionList, rbac.ResourceRepositoryTagVulnerability) {
return
}
res, err := scan.VulnListByDigest(digest)
if err != nil {
log.Errorf("Failed to get vulnerability list for image: %s:%s", repository, tag)
}
ra.Data["json"] = res
ra.ServeJSON()
}
func getSignatures(username, repository string) (map[string][]notarymodel.Target, error) {
targets, err := notary.GetInternalTargets(config.InternalNotaryEndpoint(),
username, repository)
@ -1079,33 +1036,19 @@ func (ra *RepositoryAPI) checkExistence(repository, tag string) (bool, string, e
return true, digest, nil
}
// will return nil when it failed to get data. The parm "tag" is for logging only.
func getScanOverview(digest string, tag string) *models.ImgScanOverview {
if len(digest) == 0 {
log.Debug("digest is nil")
return nil
func getSummary(pid int64, repository string, digest string) map[string]interface{} {
// At present, only get harbor native report as default behavior.
artifact := &v1.Artifact{
NamespaceID: pid,
Repository: repository,
Digest: digest,
MimeType: v1.MimeTypeDockerArtifact,
}
data, err := dao.GetImgScanOverview(digest)
sum, err := scan.DefaultController.GetSummary(artifact, []string{v1.MimeTypeNativeReport})
if err != nil {
log.Errorf("Failed to get scan result for tag:%s, digest: %s, error: %v", tag, digest, err)
logger.Errorf("Failed to get scan report summary with error: %s", err)
}
if data == nil {
return nil
}
job, err := dao.GetScanJob(data.JobID)
if err != nil {
log.Errorf("Failed to get scan job for id:%d, error: %v", data.JobID, err)
return nil
} else if job == nil { // job does not exist
log.Errorf("The scan job with id: %d does not exist, returning nil", data.JobID)
return nil
}
data.Status = job.Status
if data.Status != models.JobFinished {
log.Debugf("Unsetting vulnerable related historical values, job status: %s", data.Status)
data.Sev = 0
data.CompOverview = nil
data.DetailsKey = ""
}
return data
return sum
}

2
src/core/api/repository_test.go
View File

@ -42,7 +42,7 @@ func TestGetRepos(t *testing.T) {
} else {
assert.Equal(int(200), code, "response code should be 200")
if repos, ok := repositories.([]repoResp); ok {
assert.Equal(int(1), len(repos), "the length of repositories should be 1")
require.Equal(t, int(1), len(repos), "the length of repositories should be 1")
assert.Equal(repos[0].Name, "library/hello-world", "unexpected repository name")
} else {
t.Error("unexpected response")

192
src/core/api/scan.go Normal file
View File

@ -0,0 +1,192 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"net/http"
"strconv"
"github.com/goharbor/harbor/src/pkg/scan/report"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/utils"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/pkg/errors"
)
var digestFunc digestGetter = getDigest
// ScanAPI handles the scan related actions
type ScanAPI struct {
BaseController
// Target artifact
artifact *v1.Artifact
// Project reference
pro *models.Project
}
// Prepare sth. for the subsequent actions
func (sa *ScanAPI) Prepare() {
// Call super prepare method
sa.BaseController.Prepare()
// Parse parameters
repoName := sa.GetString(":splat")
tag := sa.GetString(":tag")
projectName, _ := utils.ParseRepository(repoName)
pro, err := sa.ProjectMgr.Get(projectName)
if err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scan API: prepare"))
return
}
if pro == nil {
sa.SendNotFoundError(errors.Errorf("project %s not found", projectName))
return
}
sa.pro = pro
// Check authentication
if !sa.RequireAuthenticated() {
return
}
// Assemble artifact object
digest, err := digestFunc(repoName, tag, sa.SecurityCtx.GetUsername())
if err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scan API: prepare"))
return
}
sa.artifact = &v1.Artifact{
NamespaceID: pro.ProjectID,
Repository: repoName,
Tag: tag,
Digest: digest,
MimeType: v1.MimeTypeDockerArtifact,
}
logger.Debugf("Scan API receives artifact: %#v", sa.artifact)
}
// Scan artifact
func (sa *ScanAPI) Scan() {
// Check access permissions
if !sa.RequireProjectAccess(sa.pro.ProjectID, rbac.ActionCreate, rbac.ResourceScan) {
return
}
if err := scan.DefaultController.Scan(sa.artifact); err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scan API: scan"))
return
}
sa.Ctx.ResponseWriter.WriteHeader(http.StatusAccepted)
}
// Report returns the required reports with the given mime types.
func (sa *ScanAPI) Report() {
// Check access permissions
if !sa.RequireProjectAccess(sa.pro.ProjectID, rbac.ActionRead, rbac.ResourceScan) {
return
}
// Extract mime types
producesMimes := make([]string, 0)
if hl, ok := sa.Ctx.Request.Header[v1.HTTPAcceptHeader]; ok && len(hl) > 0 {
producesMimes = append(producesMimes, hl...)
}
// Get the reports
reports, err := scan.DefaultController.GetReport(sa.artifact, producesMimes)
if err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scan API: get report"))
return
}
vulItems := make(map[string]interface{})
for _, rp := range reports {
// Resolve scan report data only when it is ready
if len(rp.Report) == 0 {
continue
}
vrp, err := report.ResolveData(rp.MimeType, []byte(rp.Report))
if err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scan API: get report"))
return
}
vulItems[rp.MimeType] = vrp
}
sa.Data["json"] = vulItems
sa.ServeJSON()
}
// Log returns the log stream
func (sa *ScanAPI) Log() {
// Check access permissions
if !sa.RequireProjectAccess(sa.pro.ProjectID, rbac.ActionRead, rbac.ResourceScan) {
return
}
uuid := sa.GetString(":uuid")
bytes, err := scan.DefaultController.GetScanLog(uuid)
if err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scan API: log"))
return
}
if bytes == nil {
// Not found
sa.SendNotFoundError(errors.Errorf("report with uuid %s does not exist", uuid))
return
}
sa.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(bytes)))
sa.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain")
_, err = sa.Ctx.ResponseWriter.Write(bytes)
if err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scan API: log"))
}
}
// digestGetter is a function template for getting digest.
// TODO: This can be removed if the registry access interface is ready.
type digestGetter func(repo, tag string, username string) (string, error)
func getDigest(repo, tag string, username string) (string, error) {
client, err := coreutils.NewRepositoryClientForUI(username, repo)
if err != nil {
return "", err
}
digest, exists, err := client.ManifestExist(tag)
if err != nil {
return "", err
}
if !exists {
return "", errors.Errorf("tag %s does exist", tag)
}
return digest, nil
}

82
src/core/api/scan_job.go
View File

@ -1,82 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/utils"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
)
// ScanJobAPI handles request to /api/scanJobs/:id/log
type ScanJobAPI struct {
BaseController
jobID int64
projectName string
jobUUID string
}
// Prepare validates that whether user has read permission to the project of the repo the scan job scanned.
func (sj *ScanJobAPI) Prepare() {
sj.BaseController.Prepare()
if !sj.SecurityCtx.IsAuthenticated() {
sj.SendUnAuthorizedError(errors.New("UnAuthorized"))
return
}
id, err := sj.GetInt64FromPath(":id")
if err != nil {
sj.SendBadRequestError(errors.New("invalid ID"))
return
}
sj.jobID = id
data, err := dao.GetScanJob(id)
if err != nil {
log.Errorf("Failed to load job data for job: %d, error: %v", id, err)
sj.SendInternalServerError(errors.New("Failed to get Job data"))
return
}
projectName := strings.SplitN(data.Repository, "/", 2)[0]
if !sj.RequireProjectAccess(projectName, rbac.ActionRead, rbac.ResourceRepositoryTagScanJob) {
log.Errorf("User does not have read permission for project: %s", projectName)
return
}
sj.projectName = projectName
sj.jobUUID = data.UUID
}
// GetLog ...
func (sj *ScanJobAPI) GetLog() {
logBytes, err := utils.GetJobServiceClient().GetJobLog(sj.jobUUID)
if err != nil {
sj.ParseAndHandleError(fmt.Sprintf("Failed to get job logs, uuid: %s, error: %v", sj.jobUUID, err), err)
return
}
sj.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(logBytes)))
sj.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain")
_, err = sj.Ctx.ResponseWriter.Write(logBytes)
if err != nil {
sj.SendInternalServerError(fmt.Errorf("Failed to write job logs, uuid: %s, error: %v", sj.jobUUID, err))
}
}

214
src/core/api/scan_test.go Normal file
View File

@ -0,0 +1,214 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"fmt"
"net/http"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
dscan "github.com/goharbor/harbor/src/pkg/scan/dao/scan"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
var scanBaseURL = "/api/repositories/library/hello-world/tags/latest/scan"
// ScanAPITestSuite is the test suite for scan API.
type ScanAPITestSuite struct {
suite.Suite
originalC scan.Controller
c *MockScanAPIController
originalDigestGetter digestGetter
artifact *v1.Artifact
}
// TestScanAPI is the entry point of ScanAPITestSuite.
func TestScanAPI(t *testing.T) {
suite.Run(t, new(ScanAPITestSuite))
}
// SetupSuite prepares test env for suite.
func (suite *ScanAPITestSuite) SetupSuite() {
suite.artifact = &v1.Artifact{
NamespaceID: (int64)(1),
Repository: "library/hello-world",
Tag: "latest",
Digest: "digest-code-001",
MimeType: v1.MimeTypeDockerArtifact,
}
}
// SetupTest prepares test env for test cases.
func (suite *ScanAPITestSuite) SetupTest() {
suite.originalC = scan.DefaultController
suite.c = &MockScanAPIController{}
scan.DefaultController = suite.c
suite.originalDigestGetter = digestFunc
digestFunc = func(repo, tag string, username string) (s string, e error) {
return "digest-code-001", nil
}
}
// TearDownTest ...
func (suite *ScanAPITestSuite) TearDownTest() {
scan.DefaultController = suite.originalC
digestFunc = suite.originalDigestGetter
}
// TestScanAPIBase ...
func (suite *ScanAPITestSuite) TestScanAPIBase() {
suite.c.On("Scan", &v1.Artifact{}).Return(nil)
// Including general cases
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
url: scanBaseURL,
method: http.MethodGet,
},
code: http.StatusUnauthorized,
},
// 403
{
request: &testingRequest{
url: scanBaseURL,
method: http.MethodPost,
credential: projGuest,
},
code: http.StatusForbidden,
},
}
runCodeCheckingCases(suite.T(), cases...)
}
// TestScanAPIScan ...
func (suite *ScanAPITestSuite) TestScanAPIScan() {
suite.c.On("Scan", suite.artifact).Return(nil)
// Including general cases
cases := []*codeCheckingCase{
// 202
{
request: &testingRequest{
url: scanBaseURL,
method: http.MethodPost,
credential: projDeveloper,
},
code: http.StatusAccepted,
},
}
runCodeCheckingCases(suite.T(), cases...)
}
// TestScanAPIReport ...
func (suite *ScanAPITestSuite) TestScanAPIReport() {
suite.c.On("GetReport", suite.artifact, []string{v1.MimeTypeNativeReport}).Return([]*dscan.Report{}, nil)
vulItems := make(map[string]interface{})
header := make(http.Header)
header.Add("Accept", v1.MimeTypeNativeReport)
err := handleAndParse(
&testingRequest{
url: scanBaseURL,
method: http.MethodGet,
credential: projDeveloper,
header: header,
}, &vulItems)
require.NoError(suite.T(), err)
}
// TestScanAPILog ...
func (suite *ScanAPITestSuite) TestScanAPILog() {
suite.c.On("GetScanLog", "the-uuid-001").Return([]byte(`{"log": "this is my log"}`), nil)
logs := make(map[string]string)
err := handleAndParse(
&testingRequest{
url: fmt.Sprintf("%s/%s", scanBaseURL, "the-uuid-001/log"),
method: http.MethodGet,
credential: projDeveloper,
}, &logs)
require.NoError(suite.T(), err)
assert.Condition(suite.T(), func() (success bool) {
success = len(logs) > 0
return
})
}
// Mock things
// MockScanAPIController ...
type MockScanAPIController struct {
mock.Mock
}
// Scan ...
func (msc *MockScanAPIController) Scan(artifact *v1.Artifact) error {
args := msc.Called(artifact)
return args.Error(0)
}
func (msc *MockScanAPIController) GetReport(artifact *v1.Artifact, mimeTypes []string) ([]*dscan.Report, error) {
args := msc.Called(artifact, mimeTypes)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).([]*dscan.Report), args.Error(1)
}
func (msc *MockScanAPIController) GetSummary(artifact *v1.Artifact, mimeTypes []string) (map[string]interface{}, error) {
args := msc.Called(artifact, mimeTypes)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(map[string]interface{}), args.Error(1)
}
func (msc *MockScanAPIController) GetScanLog(uuid string) ([]byte, error) {
args := msc.Called(uuid)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).([]byte), args.Error(1)
}
func (msc *MockScanAPIController) HandleJobHooks(trackID string, change *job.StatusChange) error {
args := msc.Called(trackID, change)
return args.Error(0)
}

78
src/core/api/scanners.go
View File

@ -62,6 +62,21 @@ func (sa *ScannerAPI) Get() {
}
}
// Metadata returns the metadata of the given scanner.
func (sa *ScannerAPI) Metadata() {
uuid := sa.GetStringFromPath(":uuid")
meta, err := sa.c.GetMetadata(uuid)
if err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scanner API: get metadata"))
return
}
// Response to the client
sa.Data["json"] = meta
sa.ServeJSON()
}
// List all the scanners
func (sa *ScannerAPI) List() {
p, pz, err := sa.GetPaginationParams()
@ -77,7 +92,7 @@ func (sa *ScannerAPI) List() {
// Get query key words
kws := make(map[string]interface{})
properties := []string{"name", "description", "url"}
properties := []string{"name", "description", "url", "ex_name", "ex_url"}
for _, k := range properties {
kw := sa.GetString(k)
if len(kw) > 0 {
@ -193,10 +208,6 @@ func (sa *ScannerAPI) Update() {
// Delete the scanner
func (sa *ScannerAPI) Delete() {
uid := sa.GetStringFromPath(":uuid")
if len(uid) == 0 {
sa.SendBadRequestError(errors.New("missing uid"))
return
}
deleted, err := sa.c.DeleteRegistration(uid)
if err != nil {
@ -217,10 +228,6 @@ func (sa *ScannerAPI) Delete() {
// SetAsDefault sets the given registration as default one
func (sa *ScannerAPI) SetAsDefault() {
uid := sa.GetStringFromPath(":uuid")
if len(uid) == 0 {
sa.SendBadRequestError(errors.New("missing uid"))
return
}
m := make(map[string]interface{})
if err := sa.DecodeJSONReq(&m); err != nil {
@ -242,51 +249,22 @@ func (sa *ScannerAPI) SetAsDefault() {
sa.SendForbiddenError(errors.Errorf("not supported: %#v", m))
}
// GetProjectScanner gets the project level scanner
func (sa *ScannerAPI) GetProjectScanner() {
pid, err := sa.GetInt64FromPath(":pid")
if err != nil {
sa.SendBadRequestError(errors.Wrap(err, "scanner API: get project scanners"))
// Ping the registration.
func (sa *ScannerAPI) Ping() {
r := &scanner.Registration{}
if err := sa.DecodeJSONReq(r); err != nil {
sa.SendBadRequestError(errors.Wrap(err, "scanner API: ping"))
return
}
r, err := sa.c.GetRegistrationByProject(pid)
if err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scanner API: get project scanners"))
if err := r.Validate(false); err != nil {
sa.SendBadRequestError(errors.Wrap(err, "scanner API: ping"))
return
}
if r != nil {
sa.Data["json"] = r
} else {
sa.Data["json"] = make(map[string]interface{})
}
sa.ServeJSON()
}
// SetProjectScanner sets the project level scanner
func (sa *ScannerAPI) SetProjectScanner() {
pid, err := sa.GetInt64FromPath(":pid")
if err != nil {
sa.SendBadRequestError(errors.Wrap(err, "scanner API: set project scanners"))
return
}
body := make(map[string]string)
if err := sa.DecodeJSONReq(&body); err != nil {
sa.SendBadRequestError(errors.Wrap(err, "scanner API: set project scanners"))
return
}
uuid, ok := body["uuid"]
if !ok || len(uuid) == 0 {
sa.SendBadRequestError(errors.New("missing scanner uuid when setting project scanner"))
return
}
if err := sa.c.SetRegistrationByProject(pid, uuid); err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scanner API: set project scanners"))
if _, err := sa.c.Ping(r); err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scanner API: ping"))
return
}
}
@ -294,10 +272,6 @@ func (sa *ScannerAPI) SetProjectScanner() {
// get the specified scanner
func (sa *ScannerAPI) get() *scanner.Registration {
uid := sa.GetStringFromPath(":uuid")
if len(uid) == 0 {
sa.SendBadRequestError(errors.New("missing uid"))
return nil
}
r, err := sa.c.GetRegistration(uid)
if err != nil {

63
src/core/api/scanners_test.go
View File

@ -19,6 +19,8 @@ import (
"net/http"
"testing"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/q"
sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
@ -256,45 +258,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPISetDefault() {
})
}
// TestScannerAPIProjectScanner tests the API of getting/setting project level scanner
func (suite *ScannerAPITestSuite) TestScannerAPIProjectScanner() {
suite.mockC.On("SetRegistrationByProject", int64(1), "uuid").Return(nil)
// Set
body := make(map[string]interface{}, 1)
body["uuid"] = "uuid"
runCodeCheckingCases(suite.T(), &codeCheckingCase{
request: &testingRequest{
url: fmt.Sprintf("/api/projects/%d/scanner", 1),
method: http.MethodPut,
credential: sysAdmin,
bodyJSON: body,
},
code: http.StatusOK,
})
r := &scanner.Registration{
ID: 1004,
UUID: "uuid",
Name: "TestScannerAPIProjectScanner",
Description: "JUST FOR TEST",
URL: "https://a.b.c",
}
suite.mockC.On("GetRegistrationByProject", int64(1)).Return(r, nil)
// Get
rr := &scanner.Registration{}
err := handleAndParse(&testingRequest{
url: fmt.Sprintf("/api/projects/%d/scanner", 1),
method: http.MethodGet,
credential: sysAdmin,
}, rr)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), r.Name, rr.Name)
assert.Equal(suite.T(), r.UUID, rr.UUID)
}
func (suite *ScannerAPITestSuite) mockQuery(r *scanner.Registration) {
kw := make(map[string]interface{}, 1)
kw["name"] = r.Name
@ -385,3 +348,25 @@ func (m *MockScannerAPIController) GetRegistrationByProject(projectID int64) (*s
return s.(*scanner.Registration), args.Error(1)
}
// Ping ...
func (m *MockScannerAPIController) Ping(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) {
args := m.Called(registration)
sam := args.Get(0)
if sam == nil {
return nil, args.Error(1)
}
return sam.(*v1.ScannerAdapterMetadata), nil
}
// GetMetadata ...
func (m *MockScannerAPIController) GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error) {
args := m.Called(registrationUUID)
sam := args.Get(0)
if sam == nil {
return nil, args.Error(1)
}
return sam.(*v1.ScannerAdapterMetadata), nil
}

41
src/core/api/user.go
View File

@ -52,7 +52,7 @@ type userSearch struct {
Username string `json:"username"`
}
type secretResp struct {
type secretReq struct {
Secret string `json:"secret"`
}
@ -405,8 +405,8 @@ func (ua *UserAPI) ChangePassword() {
return
}
if len(req.NewPassword) == 0 {
ua.SendBadRequestError(errors.New("empty new_password"))
if err := validateSecret(req.NewPassword); err != nil {
ua.SendBadRequestError(err)
return
}
@ -512,8 +512,8 @@ func (ua *UserAPI) ListUserPermissions() {
return
}
// GenCLISecret generates a new CLI secret and replace the old one
func (ua *UserAPI) GenCLISecret() {
// SetCLISecret handles request PUT /api/users/:id/cli_secret to update the CLI secret of the user
func (ua *UserAPI) SetCLISecret() {
if ua.AuthMode != common.OIDCAuth {
ua.SendPreconditionFailedError(errors.New("the auth mode has to be oidc auth"))
return
@ -534,8 +534,17 @@ func (ua *UserAPI) GenCLISecret() {
return
}
sec := utils.GenerateRandomString()
encSec, err := utils.ReversibleEncrypt(sec, ua.secretKey)
s := &secretReq{}
if err := ua.DecodeJSONReq(s); err != nil {
ua.SendBadRequestError(err)
return
}
if err := validateSecret(s.Secret); err != nil {
ua.SendBadRequestError(err)
return
}
encSec, err := utils.ReversibleEncrypt(s.Secret, ua.secretKey)
if err != nil {
log.Errorf("Failed to encrypt secret, error: %v", err)
ua.SendInternalServerError(errors.New("failed to encrypt secret"))
@ -548,8 +557,6 @@ func (ua *UserAPI) GenCLISecret() {
ua.SendInternalServerError(errors.New("failed to update secret in DB"))
return
}
ua.Data["json"] = secretResp{sec}
ua.ServeJSON()
}
func (ua *UserAPI) getOIDCUserInfo() (*models.OIDCUser, error) {
@ -588,12 +595,24 @@ func validate(user models.User) error {
if utils.IsContainIllegalChar(user.Username, []string{",", "~", "#", "$", "%"}) {
return fmt.Errorf("username contains illegal characters")
}
if utils.IsIllegalLength(user.Password, 8, 20) {
return fmt.Errorf("password with illegal length")
if err := validateSecret(user.Password); err != nil {
return err
}
return commonValidate(user)
}
func validateSecret(in string) error {
hasLower := regexp.MustCompile(`[a-z]`)
hasUpper := regexp.MustCompile(`[A-Z]`)
hasNumber := regexp.MustCompile(`[0-9]`)
if len(in) >= 8 && hasLower.MatchString(in) && hasUpper.MatchString(in) && hasNumber.MatchString(in) {
return nil
}
return errors.New("the password or secret must longer than 8 chars with at least 1 uppercase letter, 1 lowercase letter and 1 number")
}
// commonValidate validates email, realname, comment information when user register or change their profile
func commonValidate(user models.User) error {

16
src/core/api/user_test.go
View File

@ -380,8 +380,8 @@ func buildChangeUserPasswordURL(id int) string {
func TestUsersUpdatePassword(t *testing.T) {
fmt.Println("Testing Update User Password")
oldPassword := "old_password"
newPassword := "new_password"
oldPassword := "old_Passw0rd"
newPassword := "new_Passw0rd"
user01 := models.User{
Username: "user01_for_testing_change_password",
@ -515,7 +515,7 @@ func TestUsersUpdatePassword(t *testing.T) {
method: http.MethodPut,
url: buildChangeUserPasswordURL(user01.UserID),
bodyJSON: &passwordReq{
NewPassword: "another_new_password",
NewPassword: "another_new_Passw0rd",
},
credential: admin,
},
@ -642,3 +642,13 @@ func TestUsersCurrentPermissions(t *testing.T) {
assert.Nil(err)
assert.Equal(int(403), httpStatusCode, "httpStatusCode should be 403")
}
func TestValidateSecret(t *testing.T) {
assert.NotNil(t, validateSecret(""))
assert.NotNil(t, validateSecret("12345678"))
assert.NotNil(t, validateSecret("passw0rd"))
assert.NotNil(t, validateSecret("PASSW0RD"))
assert.NotNil(t, validateSecret("Sh0rt"))
assert.Nil(t, validateSecret("Passw0rd"))
assert.Nil(t, validateSecret("Thisis1Valid_password"))
}

6
src/core/middlewares/util/util.go
View File

@ -40,7 +40,7 @@ import (
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/promgr"
"github.com/goharbor/harbor/src/pkg/scan/whitelist"
"github.com/opencontainers/go-digest"
digest "github.com/opencontainers/go-digest"
)
type contextKey string
@ -346,6 +346,10 @@ func (pc PmsPolicyChecker) ContentTrustEnabled(name string) bool {
log.Errorf("Unexpected error when getting the project, error: %v", err)
return true
}
if project == nil {
log.Debugf("project %s not found", name)
return false
}
return project.ContentTrustEnabled()
}

5
src/core/middlewares/util/util_test.go
View File

@ -32,7 +32,7 @@ import (
notarytest "github.com/goharbor/harbor/src/common/utils/notary/test"
testutils "github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/core/config"
"github.com/opencontainers/go-digest"
digest "github.com/opencontainers/go-digest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -187,6 +187,9 @@ func TestPMSPolicyChecker(t *testing.T) {
assert.True(t, projectVulnerableEnabled)
assert.Equal(t, projectVulnerableSeverity, models.SevLow)
assert.Empty(t, wl.Items)
contentTrustFlag = GetPolicyChecker().ContentTrustEnabled("non_exist_project")
assert.False(t, contentTrustFlag)
}
func TestCopyResp(t *testing.T) {

20
src/core/router.go
View File

@ -52,7 +52,7 @@ func initRouters() {
beego.Router("/api/users/:id([0-9]+)/password", &api.UserAPI{}, "put:ChangePassword")
beego.Router("/api/users/:id/permissions", &api.UserAPI{}, "get:ListUserPermissions")
beego.Router("/api/users/:id/sysadmin", &api.UserAPI{}, "put:ToggleUserAdminRole")
beego.Router("/api/users/:id/gen_cli_secret", &api.UserAPI{}, "post:GenCLISecret")
beego.Router("/api/users/:id/cli_secret", &api.UserAPI{}, "put:SetCLISecret")
beego.Router("/api/usergroups/?:ugid([0-9]+)", &api.UserGroupAPI{})
beego.Router("/api/ldap/ping", &api.LdapAPI{}, "post:Ping")
beego.Router("/api/ldap/users/search", &api.LdapAPI{}, "get:Search")
@ -87,12 +87,9 @@ func initRouters() {
beego.Router("/api/repositories/*/tags/:tag/labels", &api.RepositoryLabelAPI{}, "get:GetOfImage;post:AddToImage")
beego.Router("/api/repositories/*/tags/:tag/labels/:id([0-9]+)", &api.RepositoryLabelAPI{}, "delete:RemoveFromImage")
beego.Router("/api/repositories/*/tags", &api.RepositoryAPI{}, "get:GetTags;post:Retag")
beego.Router("/api/repositories/*/tags/:tag/scan", &api.RepositoryAPI{}, "post:ScanImage")
beego.Router("/api/repositories/*/tags/:tag/vulnerability/details", &api.RepositoryAPI{}, "Get:VulnerabilityDetails")
beego.Router("/api/repositories/*/tags/:tag/manifest", &api.RepositoryAPI{}, "get:GetManifests")
beego.Router("/api/repositories/*/signatures", &api.RepositoryAPI{}, "get:GetSignatures")
beego.Router("/api/repositories/top", &api.RepositoryAPI{}, "get:GetTopRepos")
beego.Router("/api/jobs/scan/:id([0-9]+)/log", &api.ScanJobAPI{}, "get:GetLog")
beego.Router("/api/system/gc", &api.GCAPI{}, "get:List")
beego.Router("/api/system/gc/:id", &api.GCAPI{}, "get:GetGC")
@ -142,7 +139,6 @@ func initRouters() {
// external service that hosted on harbor process:
beego.Router("/service/notifications", &registry.NotificationHandler{})
beego.Router("/service/notifications/jobs/scan/:id([0-9]+)", &jobs.Handler{}, "post:HandleScan")
beego.Router("/service/notifications/jobs/adminjob/:id([0-9]+)", &admin.Handler{}, "post:HandleAdminJob")
beego.Router("/service/notifications/jobs/replication/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationScheduleJob")
beego.Router("/service/notifications/jobs/replication/task/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationTask")
@ -201,8 +197,20 @@ func initRouters() {
scannerAPI := &api.ScannerAPI{}
beego.Router("/api/scanners", scannerAPI, "post:Create;get:List")
beego.Router("/api/scanners/:uuid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault")
beego.Router("/api/scanners/:uuid/metadata", scannerAPI, "get:Metadata")
beego.Router("/api/scanners/ping", scannerAPI, "post:Ping")
// Add routes for project level scanner
beego.Router("/api/projects/:pid([0-9]+)/scanner", scannerAPI, "get:GetProjectScanner;put:SetProjectScanner")
proScannerAPI := &api.ProjectScannerAPI{}
beego.Router("/api/projects/:pid([0-9]+)/scanner", proScannerAPI, "get:GetProjectScanner;put:SetProjectScanner")
// Add routes for scan
scanAPI := &api.ScanAPI{}
beego.Router("/api/repositories/*/tags/:tag/scan", scanAPI, "post:Scan;get:Report")
beego.Router("/api/repositories/*/tags/:tag/scan/:uuid/log", scanAPI, "get:Log")
// Handle scan hook
beego.Router("/service/notifications/jobs/scan/:uuid", &jobs.Handler{}, "post:HandleScan")
// Error pages
beego.ErrorController(&controllers.ErrorController{})

35
src/core/service/notifications/jobs/handler.go
View File

@ -18,7 +18,8 @@ import (
"encoding/json"
"time"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
"github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
@ -49,29 +50,36 @@ type Handler struct {
rawStatus string
checkIn string
revision int64
trackID string
change *jjob.StatusChange
}
// Prepare ...
func (h *Handler) Prepare() {
id, err := h.GetInt64FromPath(":id")
if err != nil {
log.Errorf("Failed to get job ID, error: %v", err)
// Avoid job service from resending...
h.Abort("200")
return
h.trackID = h.GetStringFromPath(":uuid")
if len(h.trackID) == 0 {
id, err := h.GetInt64FromPath(":id")
if err != nil {
log.Errorf("Failed to get job ID, error: %v", err)
// Avoid job service from resending...
h.Abort("200")
return
}
h.id = id
}
h.id = id
var data jjob.StatusChange
err = json.Unmarshal(h.Ctx.Input.CopyBody(1<<32), &data)
err := json.Unmarshal(h.Ctx.Input.CopyBody(1<<32), &data)
if err != nil {
log.Errorf("Failed to decode job status change, job ID: %d, error: %v", id, err)
log.Errorf("Failed to decode job status change with error: %v", err)
h.Abort("200")
return
}
h.change = &data
h.rawStatus = data.Status
status, ok := statusMap[data.Status]
if !ok {
log.Debugf("drop the job status update event: job id-%d, status-%s", id, status)
log.Debugf("drop the job status update event: job id-%d/track id-%s, status-%s", h.id, h.trackID, status)
h.Abort("200")
return
}
@ -84,7 +92,8 @@ func (h *Handler) Prepare() {
// HandleScan handles the webhook of scan job
func (h *Handler) HandleScan() {
log.Debugf("received san job status update event: job-%d, status-%s", h.id, h.status)
log.Debugf("received san job status update event: job-%d, status-%s, track_id-%s", h.id, h.status, h.trackID)
// Trigger image scan webhook event only for JobFinished and JobError status
if h.status == models.JobFinished || h.status == models.JobError {
e := &event.Event{}
@ -101,7 +110,7 @@ func (h *Handler) HandleScan() {
}
}
if err := dao.UpdateScanJobStatus(h.id, h.status); err != nil {
if err := scan.DefaultController.HandleJobHooks(h.trackID, h.change); err != nil {
log.Errorf("Failed to update job status, id: %d, status: %s", h.id, h.status)
h.SendInternalServerError(err)
return

2
src/jobservice/job/impl/replication/replication.go
View File

@ -44,6 +44,8 @@ import (
_ "github.com/goharbor/harbor/src/replication/adapter/aliacr"
// register the Jfrog Artifactory adapter
_ "github.com/goharbor/harbor/src/replication/adapter/jfrog"
// register the Quay.io adapter
_ "github.com/goharbor/harbor/src/replication/adapter/quayio"
// register the Helm Hub adapter
_ "github.com/goharbor/harbor/src/replication/adapter/helmhub"
)

8
src/jobservice/runtime/bootstrap.go
View File

@ -23,9 +23,6 @@ import (
"syscall"
"time"
"github.com/gomodule/redigo/redis"
"github.com/pkg/errors"
"github.com/goharbor/harbor/src/jobservice/api"
"github.com/goharbor/harbor/src/jobservice/common/utils"
"github.com/goharbor/harbor/src/jobservice/config"
@ -45,7 +42,10 @@ import (
"github.com/goharbor/harbor/src/jobservice/worker"
"github.com/goharbor/harbor/src/jobservice/worker/cworker"
"github.com/goharbor/harbor/src/pkg/retention"
sc "github.com/goharbor/harbor/src/pkg/scan"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/gomodule/redigo/redis"
"github.com/pkg/errors"
)
const (
@ -242,7 +242,7 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool(
// Only for debugging and testing purpose
job.SampleJob: (*sample.Job)(nil),
// Functional jobs
job.ImageScanJob: (*scan.ClairJob)(nil),
job.ImageScanJob: (*sc.Job)(nil),
job.ImageScanAllJob: (*scan.All)(nil),
job.ImageGC: (*gc.GarbageCollector)(nil),
job.Replication: (*replication.Replication)(nil),

379
src/pkg/scan/api/scan/base_controller.go
View File

@ -15,44 +15,397 @@
package scan
import (
"fmt"
"time"
"github.com/goharbor/harbor/src/common"
cj "github.com/goharbor/harbor/src/common/job"
jm "github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/robot"
"github.com/goharbor/harbor/src/pkg/robot/model"
sca "github.com/goharbor/harbor/src/pkg/scan"
sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/report"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/google/uuid"
"github.com/pkg/errors"
)
// DefaultController is a default singleton scan API controller.
var DefaultController = NewController()
const (
configRegistryEndpoint = "registryEndpoint"
configCoreInternalAddr = "coreInternalAddr"
)
// uuidGenerator is a func template which is for generating UUID.
type uuidGenerator func() (string, error)
// configGetter is a func template which is used to wrap the config management
// utility methods.
type configGetter func(cfg string) (string, error)
// basicController is default implementation of api.Controller interface
type basicController struct {
// Client for talking to scanner adapter
client v1.Client
// Manage the scan report records
manager report.Manager
// Scanner controller
sc sc.Controller
// Robot account controller
rc robot.Controller
// Job service client
jc cj.Client
// UUID generator
uuid uuidGenerator
// Configuration getter func
config configGetter
}
// NewController news a scan API controller
func NewController() Controller {
return &basicController{}
return &basicController{
// New report manager
manager: report.NewManager(),
// Refer to the default scanner controller
sc: sc.DefaultController,
// Refer to the default robot account controller
rc: robot.RobotCtr,
// Refer to the default job service client
jc: cj.GlobalClient,
// Generate UUID with uuid lib
uuid: func() (string, error) {
aUUID, err := uuid.NewUUID()
if err != nil {
return "", err
}
return aUUID.String(), nil
},
// Get the required configuration options
config: func(cfg string) (string, error) {
switch cfg {
case configRegistryEndpoint:
return config.ExtEndpoint()
case configCoreInternalAddr:
return config.InternalCoreURL(), nil
default:
return "", errors.Errorf("configuration option %s not defined", cfg)
}
},
}
}
// Scan ...
func (bc *basicController) Scan(artifact *v1.Artifact) error {
if artifact == nil {
return errors.New("nil artifact to scan")
}
r, err := bc.sc.GetRegistrationByProject(artifact.NamespaceID)
if err != nil {
return errors.Wrap(err, "scan controller: scan")
}
// Check the health of the registration by ping.
// The metadata of the scanner adapter is also returned.
meta, err := bc.sc.Ping(r)
if err != nil {
return errors.Wrap(err, "scan controller: scan")
}
// Generate a UUID as track ID which groups the report records generated
// by the specified registration for the digest with given mime type.
trackID, err := bc.uuid()
if err != nil {
return errors.Wrap(err, "scan controller: scan")
}
producesMimes := make([]string, 0)
matched := false
for _, ca := range meta.Capabilities {
for _, cm := range ca.ConsumesMimeTypes {
if cm == artifact.MimeType {
matched = true
break
}
}
if matched {
for _, pm := range ca.ProducesMimeTypes {
// Create report placeholder first
reportPlaceholder := &scan.Report{
Digest: artifact.Digest,
RegistrationUUID: r.UUID,
Status: job.PendingStatus.String(),
StatusCode: job.PendingStatus.Code(),
TrackID: trackID,
MimeType: pm,
}
_, e := bc.manager.Create(reportPlaceholder)
if e != nil {
// Recorded by error wrap and logged at the same time.
if err == nil {
err = e
} else {
err = errors.Wrap(e, err.Error())
}
logger.Error(errors.Wrap(e, "scan controller: scan"))
continue
}
producesMimes = append(producesMimes, pm)
}
break
}
}
// Scanner does not support scanning the given artifact.
if !matched {
return errors.Errorf("the configured scanner %s does not support scanning artifact with mime type %s", r.Name, artifact.MimeType)
}
// If all the record are created failed.
if len(producesMimes) == 0 {
// Return the last error
return errors.Wrap(err, "scan controller: scan")
}
jobID, err := bc.launchScanJob(trackID, artifact, r, producesMimes)
if err != nil {
// Update the status to the concrete error
// Change status code to normal error code
if e := bc.manager.UpdateStatus(trackID, err.Error(), 0); e != nil {
err = errors.Wrap(e, err.Error())
}
return errors.Wrap(err, "scan controller: scan")
}
// Insert the generated job ID now
// It will not block the whole process. If any errors happened, just logged.
if err := bc.manager.UpdateScanJobID(trackID, jobID); err != nil {
logger.Error(errors.Wrap(err, "scan controller: scan"))
}
return nil
}
// GetReport ...
func (bc *basicController) GetReport(artifact *v1.Artifact) ([]*scan.Report, error) {
return nil, nil
func (bc *basicController) GetReport(artifact *v1.Artifact, mimeTypes []string) ([]*scan.Report, error) {
if artifact == nil {
return nil, errors.New("no way to get report for nil artifact")
}
mimes := make([]string, 0)
mimes = append(mimes, mimeTypes...)
if len(mimes) == 0 {
// Retrieve native as default
mimes = append(mimes, v1.MimeTypeNativeReport)
}
// Get current scanner settings
r, err := bc.sc.GetRegistrationByProject(artifact.NamespaceID)
if err != nil {
return nil, errors.Wrap(err, "scan controller: get report")
}
if r == nil {
return nil, errors.New("no scanner registration configured")
}
return bc.manager.GetBy(artifact.Digest, r.UUID, mimes)
}
// GetSummary ...
func (bc *basicController) GetSummary(artifact *v1.Artifact, mimeTypes []string) (map[string]interface{}, error) {
if artifact == nil {
return nil, errors.New("no way to get report summaries for nil artifact")
}
// Get reports first
rps, err := bc.GetReport(artifact, mimeTypes)
if err != nil {
return nil, err
}
summaries := make(map[string]interface{}, len(rps))
for _, rp := range rps {
sum, err := report.GenerateSummary(rp)
if err != nil {
return nil, err
}
summaries[rp.MimeType] = sum
}
return summaries, nil
}
// GetScanLog ...
func (bc *basicController) GetScanLog(digest string) ([]byte, error) {
return nil, nil
}
func (bc *basicController) GetScanLog(uuid string) ([]byte, error) {
if len(uuid) == 0 {
return nil, errors.New("empty uuid to get scan log")
}
// Ping ...
func (bc *basicController) Ping(registration *scanner.Registration) error {
return nil
// Get by uuid
sr, err := bc.manager.Get(uuid)
if err != nil {
return nil, errors.Wrap(err, "scan controller: get scan log")
}
if sr == nil {
// Not found
return nil, nil
}
// Not job error
if sr.StatusCode == job.ErrorStatus.Code() {
jst := job.Status(sr.Status)
if jst.Code() == -1 {
return []byte(sr.Status), nil
}
}
// Job log
return bc.jc.GetJobLog(sr.JobID)
}
// HandleJobHooks ...
func (bc *basicController) HandleJobHooks(trackID int64, change *job.StatusChange) error {
return nil
func (bc *basicController) HandleJobHooks(trackID string, change *job.StatusChange) error {
if len(trackID) == 0 {
return errors.New("empty track ID")
}
if change == nil {
return errors.New("nil change object")
}
// Check in data
if len(change.CheckIn) > 0 {
checkInReport := &sca.CheckInReport{}
if err := checkInReport.FromJSON(change.CheckIn); err != nil {
return errors.Wrap(err, "scan controller: handle job hook")
}
rpl, err := bc.manager.GetBy(
checkInReport.Digest,
checkInReport.RegistrationUUID,
[]string{checkInReport.MimeType})
if err != nil {
return errors.Wrap(err, "scan controller: handle job hook")
}
if len(rpl) == 0 {
return errors.New("no report found to update data")
}
if err := bc.manager.UpdateReportData(
rpl[0].UUID,
checkInReport.RawReport,
change.Metadata.Revision); err != nil {
return errors.Wrap(err, "scan controller: handle job hook")
}
return nil
}
return bc.manager.UpdateStatus(trackID, change.Status, change.Metadata.Revision)
}
// makeRobotAccount creates a robot account based on the arguments for scanning.
func (bc *basicController) makeRobotAccount(pid int64, repository string, ttl int64) (string, error) {
// Use uuid as name to avoid duplicated entries.
UUID, err := bc.uuid()
if err != nil {
return "", errors.Wrap(err, "scan controller: make robot account")
}
expireAt := time.Now().UTC().Add(time.Duration(ttl) * time.Second).Unix()
logger.Warningf("repository %s and expire time %d are not supported by robot controller", repository, expireAt)
resource := fmt.Sprintf("/project/%d/repository", pid)
access := []*rbac.Policy{{
Resource: rbac.Resource(resource),
Action: "pull",
}}
account := &model.RobotCreate{
Name: fmt.Sprintf("%s%s", common.RobotPrefix, UUID),
Description: "for scan",
ProjectID: pid,
Access: access,
}
rb, err := bc.rc.CreateRobotAccount(account)
if err != nil {
return "", errors.Wrap(err, "scan controller: make robot account")
}
return rb.Token, nil
}
// launchScanJob launches a job to run scan
func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact, registration *scanner.Registration, mimes []string) (jobID string, err error) {
externalURL, err := bc.config(configRegistryEndpoint)
if err != nil {
return "", errors.Wrap(err, "scan controller: launch scan job")
}
// Make a robot account with 30 minutes
robotAccount, err := bc.makeRobotAccount(artifact.NamespaceID, artifact.Repository, 1800)
if err != nil {
return "", errors.Wrap(err, "scan controller: launch scan job")
}
// Set job parameters
scanReq := &v1.ScanRequest{
Registry: &v1.Registry{
URL: externalURL,
Authorization: robotAccount,
},
Artifact: artifact,
}
rJSON, err := registration.ToJSON()
if err != nil {
return "", errors.Wrap(err, "scan controller: launch scan job")
}
sJSON, err := scanReq.ToJSON()
if err != nil {
return "", errors.Wrap(err, "launch scan job")
}
params := make(map[string]interface{})
params[sca.JobParamRegistration] = rJSON
params[sca.JobParameterRequest] = sJSON
params[sca.JobParameterMimes] = mimes
// Launch job
callbackURL, err := bc.config(configCoreInternalAddr)
if err != nil {
return "", errors.Wrap(err, "launch scan job")
}
hookURL := fmt.Sprintf("%s/service/notifications/jobs/scan/%s", callbackURL, trackID)
j := &jm.JobData{
Name: job.ImageScanJob,
Metadata: &jm.JobMetadata{
JobKind: job.KindGeneric,
},
Parameters: params,
StatusHook: hookURL,
}
return bc.jc.SubmitJob(j)
}

537
src/pkg/scan/api/scan/base_controller_test.go Normal file
View File

@ -0,0 +1,537 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"encoding/json"
"fmt"
"testing"
"time"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/pkg/robot/model"
cjm "github.com/goharbor/harbor/src/common/job/models"
jm "github.com/goharbor/harbor/src/common/job/models"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/q"
sca "github.com/goharbor/harbor/src/pkg/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
// ControllerTestSuite is the test suite for scan controller.
type ControllerTestSuite struct {
suite.Suite
registration *scanner.Registration
artifact *v1.Artifact
rawReport string
c Controller
}
// TestController is the entry point of ControllerTestSuite.
func TestController(t *testing.T) {
suite.Run(t, new(ControllerTestSuite))
}
// SetupSuite ...
func (suite *ControllerTestSuite) SetupSuite() {
suite.registration = &scanner.Registration{
ID: 1,
UUID: "uuid001",
Name: "Test-scan-controller",
URL: "http://testing.com:3128",
IsDefault: true,
}
suite.artifact = &v1.Artifact{
NamespaceID: 1,
Repository: "scan",
Tag: "golang",
Digest: "digest-code",
MimeType: v1.MimeTypeDockerArtifact,
}
m := &v1.ScannerAdapterMetadata{
Scanner: &v1.Scanner{
Name: "Clair",
Vendor: "Harbor",
Version: "0.1.0",
},
Capabilities: []*v1.ScannerCapability{{
ConsumesMimeTypes: []string{
v1.MimeTypeOCIArtifact,
v1.MimeTypeDockerArtifact,
},
ProducesMimeTypes: []string{
v1.MimeTypeNativeReport,
},
}},
Properties: v1.ScannerProperties{
"extra": "testing",
},
}
sc := &MockScannerController{}
sc.On("GetRegistrationByProject", suite.artifact.NamespaceID).Return(suite.registration, nil)
sc.On("Ping", suite.registration).Return(m, nil)
mgr := &MockReportManager{}
mgr.On("Create", &scan.Report{
Digest: "digest-code",
RegistrationUUID: "uuid001",
MimeType: "application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0",
Status: "Pending",
StatusCode: 0,
TrackID: "the-uuid-123",
}).Return("r-uuid", nil)
mgr.On("UpdateScanJobID", "the-uuid-123", "the-job-id").Return(nil)
rp := vuln.Report{
GeneratedAt: time.Now().UTC().String(),
Scanner: &v1.Scanner{
Name: "Clair",
Vendor: "Harbor",
Version: "0.1.0",
},
Severity: vuln.High,
Vulnerabilities: []*vuln.VulnerabilityItem{
{
ID: "2019-0980-0909",
Package: "dpkg",
Version: "0.9.1",
FixVersion: "0.9.2",
Severity: vuln.High,
Description: "mock one",
Links: []string{"https://vuln.com"},
},
},
}
jsonData, err := json.Marshal(rp)
require.NoError(suite.T(), err)
suite.rawReport = string(jsonData)
reports := []*scan.Report{
{
ID: 11,
UUID: "rp-uuid-001",
Digest: "digest-code",
RegistrationUUID: "uuid001",
MimeType: "application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0",
Status: "Success",
StatusCode: 3,
TrackID: "the-uuid-123",
JobID: "the-job-id",
StatusRevision: time.Now().Unix(),
Report: suite.rawReport,
StartTime: time.Now(),
EndTime: time.Now().Add(2 * time.Second),
},
}
mgr.On("GetBy", suite.artifact.Digest, suite.registration.UUID, []string{v1.MimeTypeNativeReport}).Return(reports, nil)
mgr.On("Get", "rp-uuid-001").Return(reports[0], nil)
mgr.On("UpdateReportData", "rp-uuid-001", suite.rawReport, (int64)(10000)).Return(nil)
mgr.On("UpdateStatus", "the-uuid-123", "Success", (int64)(10000)).Return(nil)
rc := &MockRobotController{}
resource := fmt.Sprintf("/project/%d/repository", suite.artifact.NamespaceID)
access := []*rbac.Policy{{
Resource: rbac.Resource(resource),
Action: "pull",
}}
rname := fmt.Sprintf("%s%s", common.RobotPrefix, "the-uuid-123")
account := &model.RobotCreate{
Name: rname,
Description: "for scan",
ProjectID: suite.artifact.NamespaceID,
Access: access,
}
rc.On("CreateRobotAccount", account).Return(&model.Robot{
ID: 1,
Name: rname,
Token: "robot-account",
Description: "for scan",
ProjectID: suite.artifact.NamespaceID,
}, nil)
// Set job parameters
req := &v1.ScanRequest{
Registry: &v1.Registry{
URL: "https://core.com",
Authorization: "robot-account",
},
Artifact: suite.artifact,
}
rJSON, err := req.ToJSON()
require.NoError(suite.T(), err)
regJSON, err := suite.registration.ToJSON()
require.NoError(suite.T(), err)
jc := &MockJobServiceClient{}
params := make(map[string]interface{})
params[sca.JobParamRegistration] = regJSON
params[sca.JobParameterRequest] = rJSON
params[sca.JobParameterMimes] = []string{v1.MimeTypeNativeReport}
j := &jm.JobData{
Name: job.ImageScanJob,
Metadata: &jm.JobMetadata{
JobKind: job.KindGeneric,
},
Parameters: params,
StatusHook: fmt.Sprintf("%s/service/notifications/jobs/scan/%s", "http://core:8080", "the-uuid-123"),
}
jc.On("SubmitJob", j).Return("the-job-id", nil)
jc.On("GetJobLog", "the-job-id").Return([]byte("job log"), nil)
suite.c = &basicController{
manager: mgr,
sc: sc,
jc: jc,
rc: rc,
uuid: func() (string, error) {
return "the-uuid-123", nil
},
config: func(cfg string) (string, error) {
switch cfg {
case configRegistryEndpoint:
return "https://core.com", nil
case configCoreInternalAddr:
return "http://core:8080", nil
}
return "", nil
},
}
}
// TearDownSuite ...
func (suite *ControllerTestSuite) TearDownSuite() {}
// TestScanControllerScan ...
func (suite *ControllerTestSuite) TestScanControllerScan() {
err := suite.c.Scan(suite.artifact)
require.NoError(suite.T(), err)
}
// TestScanControllerGetReport ...
func (suite *ControllerTestSuite) TestScanControllerGetReport() {
rep, err := suite.c.GetReport(suite.artifact, []string{v1.MimeTypeNativeReport})
require.NoError(suite.T(), err)
assert.Equal(suite.T(), 1, len(rep))
}
// TestScanControllerGetSummary ...
func (suite *ControllerTestSuite) TestScanControllerGetSummary() {
sum, err := suite.c.GetSummary(suite.artifact, []string{v1.MimeTypeNativeReport})
require.NoError(suite.T(), err)
assert.Equal(suite.T(), 1, len(sum))
}
// TestScanControllerGetScanLog ...
func (suite *ControllerTestSuite) TestScanControllerGetScanLog() {
bytes, err := suite.c.GetScanLog("rp-uuid-001")
require.NoError(suite.T(), err)
assert.Condition(suite.T(), func() (success bool) {
success = len(bytes) > 0
return
})
}
// TestScanControllerHandleJobHooks ...
func (suite *ControllerTestSuite) TestScanControllerHandleJobHooks() {
cReport := &sca.CheckInReport{
Digest: "digest-code",
RegistrationUUID: suite.registration.UUID,
MimeType: v1.MimeTypeNativeReport,
RawReport: suite.rawReport,
}
cRpJSON, err := cReport.ToJSON()
require.NoError(suite.T(), err)
statusChange := &job.StatusChange{
JobID: "the-job-id",
Status: "Success",
CheckIn: string(cRpJSON),
Metadata: &job.StatsInfo{
Revision: (int64)(10000),
},
}
err = suite.c.HandleJobHooks("the-uuid-123", statusChange)
require.NoError(suite.T(), err)
}
// Mock things
// MockReportManager ...
type MockReportManager struct {
mock.Mock
}
// Create ...
func (mrm *MockReportManager) Create(r *scan.Report) (string, error) {
args := mrm.Called(r)
return args.String(0), args.Error(1)
}
// UpdateScanJobID ...
func (mrm *MockReportManager) UpdateScanJobID(trackID string, jobID string) error {
args := mrm.Called(trackID, jobID)
return args.Error(0)
}
func (mrm *MockReportManager) UpdateStatus(trackID string, status string, rev int64) error {
args := mrm.Called(trackID, status, rev)
return args.Error(0)
}
func (mrm *MockReportManager) UpdateReportData(uuid string, report string, rev int64) error {
args := mrm.Called(uuid, report, rev)
return args.Error(0)
}
func (mrm *MockReportManager) GetBy(digest string, registrationUUID string, mimeTypes []string) ([]*scan.Report, error) {
args := mrm.Called(digest, registrationUUID, mimeTypes)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).([]*scan.Report), args.Error(1)
}
func (mrm *MockReportManager) Get(uuid string) (*scan.Report, error) {
args := mrm.Called(uuid)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(*scan.Report), args.Error(1)
}
// MockScannerController ...
type MockScannerController struct {
mock.Mock
}
// ListRegistrations ...
func (msc *MockScannerController) ListRegistrations(query *q.Query) ([]*scanner.Registration, error) {
args := msc.Called(query)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).([]*scanner.Registration), args.Error(1)
}
// CreateRegistration ...
func (msc *MockScannerController) CreateRegistration(registration *scanner.Registration) (string, error) {
args := msc.Called(registration)
return args.String(0), args.Error(1)
}
// GetRegistration ...
func (msc *MockScannerController) GetRegistration(registrationUUID string) (*scanner.Registration, error) {
args := msc.Called(registrationUUID)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(*scanner.Registration), args.Error(1)
}
// RegistrationExists ...
func (msc *MockScannerController) RegistrationExists(registrationUUID string) bool {
args := msc.Called(registrationUUID)
return args.Bool(0)
}
// UpdateRegistration ...
func (msc *MockScannerController) UpdateRegistration(registration *scanner.Registration) error {
args := msc.Called(registration)
return args.Error(0)
}
// DeleteRegistration ...
func (msc *MockScannerController) DeleteRegistration(registrationUUID string) (*scanner.Registration, error) {
args := msc.Called(registrationUUID)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(*scanner.Registration), args.Error(1)
}
// SetDefaultRegistration ...
func (msc *MockScannerController) SetDefaultRegistration(registrationUUID string) error {
args := msc.Called(registrationUUID)
return args.Error(0)
}
// SetRegistrationByProject ...
func (msc *MockScannerController) SetRegistrationByProject(projectID int64, scannerID string) error {
args := msc.Called(projectID, scannerID)
return args.Error(0)
}
// GetRegistrationByProject ...
func (msc *MockScannerController) GetRegistrationByProject(projectID int64) (*scanner.Registration, error) {
args := msc.Called(projectID)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(*scanner.Registration), args.Error(1)
}
// Ping ...
func (msc *MockScannerController) Ping(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) {
args := msc.Called(registration)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(*v1.ScannerAdapterMetadata), args.Error(1)
}
// GetMetadata ...
func (msc *MockScannerController) GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error) {
args := msc.Called(registrationUUID)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(*v1.ScannerAdapterMetadata), args.Error(1)
}
// MockJobServiceClient ...
type MockJobServiceClient struct {
mock.Mock
}
// SubmitJob ...
func (mjc *MockJobServiceClient) SubmitJob(jData *cjm.JobData) (string, error) {
args := mjc.Called(jData)
return args.String(0), args.Error(1)
}
// GetJobLog ...
func (mjc *MockJobServiceClient) GetJobLog(uuid string) ([]byte, error) {
args := mjc.Called(uuid)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).([]byte), args.Error(1)
}
// PostAction ...
func (mjc *MockJobServiceClient) PostAction(uuid, action string) error {
args := mjc.Called(uuid, action)
return args.Error(0)
}
func (mjc *MockJobServiceClient) GetExecutions(uuid string) ([]job.Stats, error) {
args := mjc.Called(uuid)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).([]job.Stats), args.Error(1)
}
// MockRobotController ...
type MockRobotController struct {
mock.Mock
}
// GetRobotAccount ...
func (mrc *MockRobotController) GetRobotAccount(id int64) (*model.Robot, error) {
args := mrc.Called(id)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(*model.Robot), args.Error(1)
}
// CreateRobotAccount ...
func (mrc *MockRobotController) CreateRobotAccount(robotReq *model.RobotCreate) (*model.Robot, error) {
args := mrc.Called(robotReq)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(*model.Robot), args.Error(1)
}
// DeleteRobotAccount ...
func (mrc *MockRobotController) DeleteRobotAccount(id int64) error {
args := mrc.Called(id)
return args.Error(0)
}
// UpdateRobotAccount ...
func (mrc *MockRobotController) UpdateRobotAccount(r *model.Robot) error {
args := mrc.Called(r)
return args.Error(0)
}
// ListRobotAccount ...
func (mrc *MockRobotController) ListRobotAccount(pid int64) ([]*model.Robot, error) {
args := mrc.Called(pid)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).([]*model.Robot), args.Error(1)
}

34
src/pkg/scan/api/scan/controller.go
View File

@ -17,7 +17,6 @@ package scan
import (
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
)
@ -25,17 +24,6 @@ import (
// TODO: Here the artifact object is reused the v1 one which is sent to the adapter,
// it should be pointed to the general artifact object in future once it's ready.
type Controller interface {
// Ping pings Scanner Adapter to test EndpointURL and Authorization settings.
// The implementation is supposed to call the GetMetadata method on scanner.Client.
// Returns `nil` if connection succeeded, a non `nil` error otherwise.
//
// Arguments:
// registration *scanner.Registration : scanner registration to ping
//
// Returns:
// error : non nil error if any errors occurred
Ping(registration *scanner.Registration) error
// Scan the given artifact
//
// Arguments:
@ -49,30 +37,42 @@ type Controller interface {
//
// Arguments:
// artifact *v1.Artifact : the scanned artifact
// mimeTypes []string : the mime types of the reports
//
// Returns:
// []*scan.Report : scan results by different scanner vendors
// error : non nil error if any errors occurred
GetReport(artifact *v1.Artifact) ([]*scan.Report, error)
GetReport(artifact *v1.Artifact, mimeTypes []string) ([]*scan.Report, error)
// GetSummary gets the summaries of the reports with given types.
//
// Arguments:
// artifact *v1.Artifact : the scanned artifact
// mimeTypes []string : the mime types of the reports
//
// Returns:
// map[string]interface{} : report summaries indexed by mime types
// error : non nil error if any errors occurred
GetSummary(artifact *v1.Artifact, mimeTypes []string) (map[string]interface{}, error)
// Get the scan log for the specified artifact with the given digest
//
// Arguments:
// digest string : the digest of the artifact
// uuid string : the UUID of the scan report
//
// Returns:
// []byte : the log text stream
// error : non nil error if any errors occurred
GetScanLog(digest string) ([]byte, error)
GetScanLog(uuid string) ([]byte, error)
// HandleJobHooks handle the hook events from the job service
// e.g : status change of the scan job or scan result
//
// Arguments:
// trackID int64 : ID for the result record
// trackID string : UUID for the report record
// change *job.StatusChange : change event from the job service
//
// Returns:
// error : non nil error if any errors occurred
HandleJobHooks(trackID int64, change *job.StatusChange) error
HandleJobHooks(trackID string, change *job.StatusChange) error
}

137
src/pkg/scan/api/scanner/base_controller.go
View File

@ -19,6 +19,7 @@ import (
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
rscanner "github.com/goharbor/harbor/src/pkg/scan/scanner"
"github.com/pkg/errors"
)
@ -35,25 +36,38 @@ func New() Controller {
return &basicController{
manager: rscanner.New(),
proMetaMgr: metamgr.NewDefaultProjectMetadataManager(),
clientPool: v1.DefaultClientPool,
}
}
// basicController is default implementation of api.Controller interface
type basicController struct {
// managers for managing the scanner registrations
// Managers for managing the scanner registrations
manager rscanner.Manager
// for operating the project level configured scanner
// For operating the project level configured scanner
proMetaMgr metamgr.ProjectMetadataManager
// Client pool for talking to adapters
clientPool v1.ClientPool
}
// ListRegistrations ...
func (bc *basicController) ListRegistrations(query *q.Query) ([]*scanner.Registration, error) {
return bc.manager.List(query)
l, err := bc.manager.List(query)
if err != nil {
return nil, errors.Wrap(err, "api controller: list registrations")
}
for _, r := range l {
_, err = bc.Ping(r)
r.Health = err == nil
}
return l, nil
}
// CreateRegistration ...
func (bc *basicController) CreateRegistration(registration *scanner.Registration) (string, error) {
// TODO: Get metadata from the adapter service first
// TODO: Check connection of the registration.
// Check if there are any registrations already existing.
l, err := bc.manager.List(&q.Query{
PageSize: 1,
@ -73,7 +87,15 @@ func (bc *basicController) CreateRegistration(registration *scanner.Registration
// GetRegistration ...
func (bc *basicController) GetRegistration(registrationUUID string) (*scanner.Registration, error) {
return bc.manager.Get(registrationUUID)
r, err := bc.manager.Get(registrationUUID)
if err != nil {
return nil, err
}
_, err = bc.Ping(r)
r.Health = err == nil
return r, nil
}
// RegistrationExists ...
@ -90,6 +112,10 @@ func (bc *basicController) RegistrationExists(registrationUUID string) bool {
// UpdateRegistration ...
func (bc *basicController) UpdateRegistration(registration *scanner.Registration) error {
if registration.IsDefault && registration.Disabled {
return errors.Errorf("default registration %s can not be marked to disabled", registration.UUID)
}
return bc.manager.Update(registration)
}
@ -162,9 +188,10 @@ func (bc *basicController) GetRegistrationByProject(projectID int64) (*scanner.R
return nil, errors.Wrap(err, "api controller: get project scanner")
}
var registration *scanner.Registration
if len(m) > 0 {
if registrationID, ok := m[proScannerMetaKey]; ok && len(registrationID) > 0 {
registration, err := bc.manager.Get(registrationID)
registration, err = bc.manager.Get(registrationID)
if err != nil {
return nil, errors.Wrap(err, "api controller: get project scanner")
}
@ -175,15 +202,103 @@ func (bc *basicController) GetRegistrationByProject(projectID int64) (*scanner.R
if err := bc.proMetaMgr.Delete(projectID, proScannerMetaKey); err != nil {
return nil, errors.Wrap(err, "api controller: get project scanner")
}
} else {
return registration, nil
}
}
}
// Second, get the default one
registration, err := bc.manager.GetDefault()
if registration == nil {
// Second, get the default one
registration, err = bc.manager.GetDefault()
if err != nil {
return nil, errors.Wrap(err, "api controller: get project scanner")
}
}
// Check status by the client later
if registration != nil {
if meta, err := bc.Ping(registration); err == nil {
registration.Scanner = meta.Scanner.Name
registration.Vendor = meta.Scanner.Vendor
registration.Version = meta.Scanner.Version
registration.Health = true
} else {
registration.Health = false
}
}
// TODO: Check status by the client later
return registration, err
}
// Ping ...
// TODO: ADD UT CASES
func (bc *basicController) Ping(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) {
if registration == nil {
return nil, errors.New("nil registration to ping")
}
client, err := bc.clientPool.Get(registration)
if err != nil {
return nil, errors.Wrap(err, "scanner controller: ping")
}
meta, err := client.GetMetadata()
if err != nil {
return nil, errors.Wrap(err, "scanner controller: ping")
}
// Validate the required properties
if meta.Scanner == nil ||
len(meta.Scanner.Name) == 0 ||
len(meta.Scanner.Version) == 0 ||
len(meta.Scanner.Vendor) == 0 {
return nil, errors.New("invalid scanner in metadata")
}
if len(meta.Capabilities) == 0 {
return nil, errors.New("invalid capabilities in metadata")
}
for _, ca := range meta.Capabilities {
// v1.MimeTypeDockerArtifact is required now
found := false
for _, cm := range ca.ConsumesMimeTypes {
if cm == v1.MimeTypeDockerArtifact {
found = true
break
}
}
if !found {
return nil, errors.Errorf("missing %s in consumes_mime_types", v1.MimeTypeDockerArtifact)
}
// v1.MimeTypeNativeReport is required
found = false
for _, pm := range ca.ProducesMimeTypes {
if pm == v1.MimeTypeNativeReport {
found = true
break
}
}
if !found {
return nil, errors.Errorf("missing %s in produces_mime_types", v1.MimeTypeNativeReport)
}
}
return meta, err
}
// GetMetadata ...
// TODO: ADD UT CASES
func (bc *basicController) GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error) {
if len(registrationUUID) == 0 {
return nil, errors.New("empty registration uuid")
}
r, err := bc.manager.Get(registrationUUID)
if err != nil {
return nil, errors.Wrap(err, "scanner controller: get metadata")
}
return bc.Ping(r)
}

82
src/pkg/scan/api/scanner/controller_test.go → src/pkg/scan/api/scanner/base_controller_test.go
View File

@ -17,6 +17,8 @@ package scanner
import (
"testing"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
@ -47,9 +49,25 @@ func (suite *ControllerTestSuite) SetupSuite() {
suite.mMgr = new(MockScannerManager)
suite.mMeta = new(MockProMetaManager)
suite.c = &basicController{
manager: suite.mMgr,
proMetaMgr: suite.mMeta,
m := &v1.ScannerAdapterMetadata{
Scanner: &v1.Scanner{
Name: "Clair",
Vendor: "Harbor",
Version: "0.1.0",
},
Capabilities: []*v1.ScannerCapability{{
ConsumesMimeTypes: []string{
v1.MimeTypeOCIArtifact,
v1.MimeTypeDockerArtifact,
},
ProducesMimeTypes: []string{
v1.MimeTypeNativeReport,
v1.MimeTypeRawReport,
},
}},
Properties: v1.ScannerProperties{
"extra": "testing",
},
}
suite.sample = &scanner.Registration{
@ -57,6 +75,17 @@ func (suite *ControllerTestSuite) SetupSuite() {
Description: "sample registration",
URL: "https://sample.scanner.com",
}
mc := &MockClient{}
mc.On("GetMetadata").Return(m, nil)
mcp := &MockClientPool{}
mcp.On("Get", suite.sample).Return(mc, nil)
suite.c = &basicController{
manager: suite.mMgr,
proMetaMgr: suite.mMeta,
clientPool: mcp,
}
}
// Clear test case
@ -282,3 +311,50 @@ func (m *MockProMetaManager) List(name, value string) ([]*models.ProjectMetadata
args := m.Called(name, value)
return args.Get(0).([]*models.ProjectMetadata), args.Error(1)
}
// MockClientPool is defined and referred by other UT cases.
type MockClientPool struct {
mock.Mock
}
// Get client
func (mcp *MockClientPool) Get(r *scanner.Registration) (v1.Client, error) {
args := mcp.Called(r)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(v1.Client), args.Error(1)
}
// MockClient is defined and referred in other UT cases.
type MockClient struct {
mock.Mock
}
// GetMetadata ...
func (mc *MockClient) GetMetadata() (*v1.ScannerAdapterMetadata, error) {
args := mc.Called()
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(*v1.ScannerAdapterMetadata), args.Error(1)
}
// SubmitScan ...
func (mc *MockClient) SubmitScan(req *v1.ScanRequest) (*v1.ScanResponse, error) {
args := mc.Called(req)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(*v1.ScanResponse), args.Error(1)
}
// GetScanReport ...
func (mc *MockClient) GetScanReport(scanRequestID, reportMIMEType string) (string, error) {
args := mc.Called(scanRequestID, reportMIMEType)
return args.String(0), args.Error(1)
}

23
src/pkg/scan/api/scanner/controller.go
View File

@ -17,6 +17,7 @@ package scanner
import (
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
)
// Controller provides the related operations of scanner for the upper API.
@ -112,4 +113,26 @@ type Controller interface {
// *scanner.Registration : the default scanner registration
// error : non nil error if any errors occurred
GetRegistrationByProject(projectID int64) (*scanner.Registration, error)
// Ping pings Scanner Adapter to test EndpointURL and Authorization settings.
// The implementation is supposed to call the GetMetadata method on scanner.Client.
// Returns `nil` if connection succeeded, a non `nil` error otherwise.
//
// Arguments:
// registration *scanner.Registration : scanner registration to ping
//
// Returns:
// *v1.ScannerAdapterMetadata : metadata returned by the scanner if successfully ping
// error : non nil error if any errors occurred
Ping(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error)
// GetMetadata returns the metadata of the given scanner.
//
// Arguments:
// registrationUUID string : the UUID of the given scanner which is marked as default
//
// Returns:
// *v1.ScannerAdapterMetadata : metadata returned by the scanner if successfully ping
// error : non nil error if any errors occurred
GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error)
}

1
src/pkg/scan/dao/scan/model.go
View File

@ -25,6 +25,7 @@ type Report struct {
RegistrationUUID string `orm:"column(registration_uuid)"`
MimeType string `orm:"column(mime_type)"`
JobID string `orm:"column(job_id)"`
TrackID string `orm:"column(track_id)"`
Status string `orm:"column(status)"`
StatusCode int `orm:"column(status_code)"`
StatusRevision int64 `orm:"column(status_rev)"`

17
src/pkg/scan/dao/scan/report.go
View File

@ -16,6 +16,7 @@ package scan
import (
"fmt"
"time"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/dao"
@ -103,7 +104,7 @@ func UpdateReportData(uuid string, report string, statusRev int64) error {
}
// UpdateReportStatus updates the report `status` with conditions matched.
func UpdateReportStatus(uuid string, status string, statusCode int, statusRev int64) error {
func UpdateReportStatus(trackID string, status string, statusCode int, statusRev int64) error {
o := dao.GetOrmer()
qt := o.QueryTable(new(Report))
@ -112,7 +113,13 @@ func UpdateReportStatus(uuid string, status string, statusCode int, statusRev in
data["status_code"] = statusCode
data["status_rev"] = statusRev
count, err := qt.Filter("uuid", uuid).
// Technically it is not correct, just to avoid changing interface and adding more code.
// running==2
if statusCode > 2 {
data["end_time"] = time.Now().UTC()
}
count, err := qt.Filter("track_id", trackID).
Filter("status_rev__lte", statusRev).
Filter("status_code__lte", statusCode).Update(data)
@ -121,20 +128,20 @@ func UpdateReportStatus(uuid string, status string, statusCode int, statusRev in
}
if count == 0 {
return errors.Errorf("no report with uuid %s updated", uuid)
return errors.Errorf("no report with track_id %s updated", trackID)
}
return nil
}
// UpdateJobID updates the report `job_id` column
func UpdateJobID(uuid string, jobID string) error {
func UpdateJobID(trackID string, jobID string) error {
o := dao.GetOrmer()
qt := o.QueryTable(new(Report))
params := make(orm.Params, 1)
params["job_id"] = jobID
_, err := qt.Filter("uuid", uuid).Update(params)
_, err := qt.Filter("track_id", trackID).Update(params)
return err
}

9
src/pkg/scan/dao/scan/report_test.go
View File

@ -45,6 +45,7 @@ func (suite *ReportTestSuite) SetupSuite() {
func (suite *ReportTestSuite) SetupTest() {
r := &Report{
UUID: "uuid",
TrackID: "track-uuid",
Digest: "digest1001",
RegistrationUUID: "ruuid",
MimeType: v1.MimeTypeNativeReport,
@ -95,7 +96,7 @@ func (suite *ReportTestSuite) TestReportList() {
// TestReportUpdateJobID tests update job ID of the report.
func (suite *ReportTestSuite) TestReportUpdateJobID() {
err := UpdateJobID("uuid", "jobid001")
err := UpdateJobID("track-uuid", "jobid001")
require.NoError(suite.T(), err)
l, err := ListReports(nil)
@ -120,12 +121,12 @@ func (suite *ReportTestSuite) TestReportUpdateReportData() {
// TestReportUpdateStatus tests update the report status.
func (suite *ReportTestSuite) TestReportUpdateStatus() {
err := UpdateReportStatus("uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 1000)
err := UpdateReportStatus("track-uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 1000)
require.NoError(suite.T(), err)
err = UpdateReportStatus("uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 900)
err = UpdateReportStatus("track-uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 900)
require.Error(suite.T(), err)
err = UpdateReportStatus("uuid", job.PendingStatus.String(), job.PendingStatus.Code(), 1000)
err = UpdateReportStatus("track-uuid", job.PendingStatus.String(), job.PendingStatus.Code(), 1000)
require.Error(suite.T(), err)
}

18
src/pkg/scan/dao/scanner/model.go
View File

@ -20,6 +20,8 @@ import (
"strings"
"time"
"github.com/goharbor/harbor/src/pkg/scan/rest/auth"
"github.com/pkg/errors"
)
@ -45,6 +47,11 @@ type Registration struct {
// Http connection settings
SkipCertVerify bool `orm:"column(skip_cert_verify);default(false)" json:"skip_certVerify"`
// Extra info about the scanner
Scanner string `orm:"-" json:"scanner,omitempty"`
Vendor string `orm:"-" json:"vendor,omitempty"`
Version string `orm:"-" json:"version,omitempty"`
// Timestamps
CreateTime time.Time `orm:"column(create_time);auto_now_add;type(datetime)" json:"create_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now;type(datetime)" json:"update_time"`
@ -89,6 +96,17 @@ func (r *Registration) Validate(checkUUID bool) error {
return errors.Wrap(err, "scanner registration validate")
}
if len(r.Auth) > 0 &&
r.Auth != auth.Basic &&
r.Auth != auth.Bearer &&
r.Auth != auth.APIKey {
return errors.Errorf("auth type %s is not supported", r.Auth)
}
if len(r.Auth) > 0 && len(r.AccessCredential) == 0 {
return errors.Errorf("access_credential is required for auth type %s", r.Auth)
}
return nil
}

45
src/pkg/scan/dao/scanner/registration.go
View File

@ -16,6 +16,7 @@ package scanner
import (
"fmt"
"strings"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/dao"
@ -93,6 +94,12 @@ func ListRegistrations(query *q.Query) ([]*Registration, error) {
if query != nil {
if len(query.Keywords) > 0 {
for k, v := range query.Keywords {
if strings.HasPrefix(k, "ex_") {
kk := strings.TrimPrefix(k, "ex_")
qt = qt.Filter(kk, v)
continue
}
qt = qt.Filter(fmt.Sprintf("%s__icontains", k), v)
}
}
@ -111,20 +118,38 @@ func ListRegistrations(query *q.Query) ([]*Registration, error) {
// SetDefaultRegistration sets the specified registration as default one
func SetDefaultRegistration(UUID string) error {
o := dao.GetOrmer()
qt := o.QueryTable(new(Registration))
_, err := qt.Filter("is_default", true).Update(orm.Params{
"is_default": false,
})
err := o.Begin()
if err != nil {
return err
}
qt2 := o.QueryTable(new(Registration))
_, err = qt2.Filter("uuid", UUID).Update(orm.Params{
"is_default": true,
})
var count int64
qt := o.QueryTable(new(Registration))
count, err = qt.Filter("uuid", UUID).
Filter("disabled", false).
Update(orm.Params{
"is_default": true,
})
if err == nil && count == 0 {
err = errors.Errorf("set default for %s failed", UUID)
}
if err == nil {
qt2 := o.QueryTable(new(Registration))
_, err = qt2.Exclude("uuid__exact", UUID).
Filter("is_default", true).
Update(orm.Params{
"is_default": false,
})
}
if err != nil {
if e := o.Rollback(); e != nil {
err = errors.Wrap(e, err.Error())
}
} else {
err = o.Commit()
}
return err
}

23
src/pkg/scan/dao/scanner/registration_test.go
View File

@ -124,6 +124,22 @@ func (suite *RegistrationDAOTestSuite) TestList() {
})
require.NoError(suite.T(), err)
require.Equal(suite.T(), 0, len(l))
// Exact match
exactKeywords := make(map[string]interface{})
exactKeywords["ex_name"] = "forUT"
l, err = ListRegistrations(&q.Query{
Keywords: exactKeywords,
})
require.NoError(suite.T(), err)
require.Equal(suite.T(), 1, len(l))
exactKeywords["ex_name"] = "forU"
l, err = ListRegistrations(&q.Query{
Keywords: exactKeywords,
})
require.NoError(suite.T(), err)
require.Equal(suite.T(), 0, len(l))
}
// TestDefault tests set/get default
@ -138,4 +154,11 @@ func (suite *RegistrationDAOTestSuite) TestDefault() {
dr, err = GetDefaultRegistration()
require.NoError(suite.T(), err)
require.NotNil(suite.T(), dr)
dr.Disabled = true
err = UpdateRegistration(dr, "disabled")
require.NoError(suite.T(), err)
err = SetDefaultRegistration(suite.registrationID)
require.Error(suite.T(), err)
}

48
src/pkg/scan/job.go
View File

@ -116,17 +116,18 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error {
// Print related infos to log
printJSONParameter(JobParamRegistration, params[JobParamRegistration].(string), myLogger)
printJSONParameter(JobParameterRequest, params[JobParameterRequest].(string), myLogger)
printJSONParameter(JobParameterRequest, removeAuthInfo(req), myLogger)
myLogger.Infof("Report mime types: %v\n", mimes)
// Submit scan request to the scanner adapter
client, err := v1.DefaultClientPool.Get(r)
if err != nil {
return errors.Wrap(err, "run scan job")
return logAndWrapError(myLogger, err, "scan job: get client")
}
resp, err := client.SubmitScan(req)
if err != nil {
return errors.Wrap(err, "run scan job")
return logAndWrapError(myLogger, err, "scan job: submit scan request")
}
// For collecting errors
@ -229,6 +230,13 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error {
return err
}
func logAndWrapError(logger logger.Interface, err error, message string) error {
e := errors.Wrap(err, message)
logger.Error(e)
return e
}
func printJSONParameter(parameter string, v string, logger logger.Interface) {
logger.Infof("%s:\n", parameter)
printPrettyJSON([]byte(v), logger)
@ -244,6 +252,23 @@ func printPrettyJSON(in []byte, logger logger.Interface) {
logger.Infof("%s\n", out.String())
}
func removeAuthInfo(sr *v1.ScanRequest) string {
req := &v1.ScanRequest{
Artifact: sr.Artifact,
Registry: &v1.Registry{
URL: sr.Registry.URL,
Authorization: "[HIDDEN]",
},
}
str, err := req.ToJSON()
if err != nil {
logger.Error(errors.Wrap(err, "scan job: remove auth"))
}
return str
}
func extractScanReq(params job.Parameters) (*v1.ScanRequest, error) {
v, ok := params[JobParameterRequest]
if !ok {
@ -263,7 +288,6 @@ func extractScanReq(params job.Parameters) (*v1.ScanRequest, error) {
if err := req.FromJSON(jsonData); err != nil {
return nil, err
}
if err := req.Validate(); err != nil {
return nil, err
}
@ -304,14 +328,24 @@ func extractMimeTypes(params job.Parameters) ([]string, error) {
return nil, errors.Errorf("missing job parameter '%s'", JobParameterMimes)
}
l, ok := v.([]string)
l, ok := v.([]interface{})
if !ok {
return nil, errors.Errorf(
"malformed job parameter '%s', expecting string but got %s",
"malformed job parameter '%s', expecting []interface{} but got %s",
JobParameterMimes,
reflect.TypeOf(v).String(),
)
}
return l, nil
mimes := make([]string, 0)
for _, v := range l {
mime, ok := v.(string)
if !ok {
return nil, errors.Errorf("expect string but got %s", reflect.TypeOf(v).String())
}
mimes = append(mimes, mime)
}
return mimes, nil
}

61
src/pkg/scan/report/base_manager.go
View File

@ -57,15 +57,21 @@ func (bm *basicManager) Create(r *scan.Report) (string, error) {
})
if err != nil {
return "", errors.Wrap(err, "check existence of report")
return "", errors.Wrap(err, "create report: check existence of report")
}
// Delete existing copy
if len(existingCopies) > 0 {
theCopy := existingCopies[0]
// Status conflict
theStatus := job.Status(theCopy.Status)
// Status is an error message
if theStatus.Code() == -1 && theCopy.StatusCode == job.ErrorStatus.Code() {
// Mark as regular error status
theStatus = job.ErrorStatus
}
// Status conflict
if theStatus.Compare(job.RunningStatus) <= 0 {
return "", errors.Errorf("conflict: a previous scanning is %s", theCopy.Status)
}
@ -73,7 +79,7 @@ func (bm *basicManager) Create(r *scan.Report) (string, error) {
// Otherwise it will be a completed report
// Clear it before insert this new one
if err := scan.DeleteReport(theCopy.UUID); err != nil {
return "", errors.Wrap(err, "clear old scan report")
return "", errors.Wrap(err, "create report: clear old scan report")
}
}
@ -91,12 +97,39 @@ func (bm *basicManager) Create(r *scan.Report) (string, error) {
// Insert
if _, err = scan.CreateReport(r); err != nil {
return "", errors.Wrap(err, "create report")
return "", errors.Wrap(err, "create report: insert")
}
return r.UUID, nil
}
// Get ...
func (bm *basicManager) Get(uuid string) (*scan.Report, error) {
if len(uuid) == 0 {
return nil, errors.New("empty uuid to get scan report")
}
kws := make(map[string]interface{})
kws["uuid"] = uuid
l, err := scan.ListReports(&q.Query{
PageNumber: 1,
PageSize: 1,
Keywords: kws,
})
if err != nil {
return nil, errors.Wrap(err, "report manager: get")
}
if len(l) == 0 {
// Not found
return nil, nil
}
return l[0], nil
}
// GetBy ...
func (bm *basicManager) GetBy(digest string, registrationUUID string, mimeTypes []string) ([]*scan.Report, error) {
if len(digest) == 0 {
@ -121,24 +154,20 @@ func (bm *basicManager) GetBy(digest string, registrationUUID string, mimeTypes
}
// UpdateScanJobID ...
func (bm *basicManager) UpdateScanJobID(uuid string, jobID string) error {
if len(uuid) == 0 || len(jobID) == 0 {
func (bm *basicManager) UpdateScanJobID(trackID string, jobID string) error {
if len(trackID) == 0 || len(jobID) == 0 {
return errors.New("bad arguments")
}
return scan.UpdateJobID(uuid, jobID)
return scan.UpdateJobID(trackID, jobID)
}
// UpdateStatus ...
func (bm *basicManager) UpdateStatus(uuid string, status string, rev int64) error {
if len(uuid) == 0 {
func (bm *basicManager) UpdateStatus(trackID string, status string, rev int64) error {
if len(trackID) == 0 {
return errors.New("missing uuid")
}
if rev <= 0 {
return errors.New("invalid data revision")
}
stCode := job.ErrorStatus.Code()
st := job.Status(status)
// Check if it is job valid status.
@ -148,7 +177,7 @@ func (bm *basicManager) UpdateStatus(uuid string, status string, rev int64) erro
stCode = st.Code()
}
return scan.UpdateReportStatus(uuid, status, stCode, rev)
return scan.UpdateReportStatus(trackID, status, stCode, rev)
}
// UpdateReportData ...
@ -157,10 +186,6 @@ func (bm *basicManager) UpdateReportData(uuid string, report string, rev int64)
return errors.New("missing uuid")
}
if rev <= 0 {
return errors.New("invalid data revision")
}
if len(report) == 0 {
return errors.New("missing report JSON data")
}

18
src/pkg/scan/report/base_manager_test.go
View File

@ -52,6 +52,7 @@ func (suite *TestManagerSuite) SetupTest() {
Digest: "d1000",
RegistrationUUID: "ruuid",
MimeType: v1.MimeTypeNativeReport,
TrackID: "tid001",
}
uuid, err := suite.m.Create(rp)
@ -70,13 +71,14 @@ func (suite *TestManagerSuite) TearDownTest() {
// TestManagerCreateWithExisting tests the case that a copy already is there when creating report.
func (suite *TestManagerSuite) TestManagerCreateWithExisting() {
err := suite.m.UpdateStatus(suite.rpUUID, job.SuccessStatus.String(), 2000)
err := suite.m.UpdateStatus("tid001", job.SuccessStatus.String(), 2000)
require.NoError(suite.T(), err)
rp := &scan.Report{
Digest: "d1000",
RegistrationUUID: "ruuid",
MimeType: v1.MimeTypeNativeReport,
TrackID: "tid002",
}
uuid, err := suite.m.Create(rp)
@ -87,6 +89,16 @@ func (suite *TestManagerSuite) TestManagerCreateWithExisting() {
suite.rpUUID = uuid
}
// TestManagerGet tests the get method.
func (suite *TestManagerSuite) TestManagerGet() {
sr, err := suite.m.Get(suite.rpUUID)
require.NoError(suite.T(), err)
require.NotNil(suite.T(), sr)
assert.Equal(suite.T(), "d1000", sr.Digest)
}
// TestManagerGetBy tests the get by method.
func (suite *TestManagerSuite) TestManagerGetBy() {
l, err := suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport})
@ -113,7 +125,7 @@ func (suite *TestManagerSuite) TestManagerUpdateJobID() {
oldJID := l[0].JobID
err = suite.m.UpdateScanJobID(suite.rpUUID, "jID1001")
err = suite.m.UpdateScanJobID("tid001", "jID1001")
require.NoError(suite.T(), err)
l, err = suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport})
@ -132,7 +144,7 @@ func (suite *TestManagerSuite) TestManagerUpdateStatus() {
oldSt := l[0].Status
err = suite.m.UpdateStatus(suite.rpUUID, job.SuccessStatus.String(), 10000)
err = suite.m.UpdateStatus("tid001", job.SuccessStatus.String(), 10000)
require.NoError(suite.T(), err)
l, err = suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport})

30
src/pkg/scan/report/manager.go
View File

@ -32,32 +32,32 @@ type Manager interface {
// Update the scan job ID of the given report.
//
// Arguments:
// uuid string : uuid to identify the report
// jobID string: scan job ID
// trackID string : uuid to identify the report
// jobID string : scan job ID
//
// Returns:
// error : non nil error if any errors occurred
//
UpdateScanJobID(uuid string, jobID string) error
UpdateScanJobID(trackID string, jobID string) error
// Update the status (mapping to the scan job status) of the given report.
//
// Arguments:
// uuid string : uuid to identify the report
// status string: status info
// rev int64 : data revision info
// trackID string : uuid to identify the report
// status string : status info
// rev int64 : data revision info
//
// Returns:
// error : non nil error if any errors occurred
//
UpdateStatus(uuid string, status string, rev int64) error
UpdateStatus(trackID string, status string, rev int64) error
// Update the report data (with JSON format) of the given report.
//
// Arguments:
// uuid string : uuid to identify the report
// report string: report JSON data
// rev int64 : data revision info
// uuid string : uuid to identify the report
// report string : report JSON data
// rev int64 : data revision info
//
// Returns:
// error : non nil error if any errors occurred
@ -77,4 +77,14 @@ type Manager interface {
// []*scan.Report : report list
// error : non nil error if any errors occurred
GetBy(digest string, registrationUUID string, mimeTypes []string) ([]*scan.Report, error)
// Get the report for the given uuid.
//
// Arguments:
// uuid string : uuid of the scan report
//
// Returns:
// *scan.Report : scan report
// error : non nil error if any errors occurred
Get(uuid string) (*scan.Report, error)
}

97
src/pkg/scan/report/summary.go Normal file
View File

@ -0,0 +1,97 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package report
import (
"reflect"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
"github.com/pkg/errors"
)
// SupportedGenerators declares mappings between mime type and summary generator func.
var SupportedGenerators = map[string]SummaryGenerator{
v1.MimeTypeNativeReport: GenerateNativeSummary,
}
// GenerateSummary is a helper function to generate report
// summary based on the given report.
func GenerateSummary(r *scan.Report) (interface{}, error) {
g, ok := SupportedGenerators[r.MimeType]
if !ok {
return nil, errors.Errorf("no generator bound with mime type %s", r.MimeType)
}
return g(r)
}
// SummaryGenerator is a func template which used to generated report
// summary for relevant mime type.
type SummaryGenerator func(r *scan.Report) (interface{}, error)
// GenerateNativeSummary generates the report summary for the native report.
func GenerateNativeSummary(r *scan.Report) (interface{}, error) {
sum := &vuln.NativeReportSummary{}
sum.ReportID = r.UUID
sum.StartTime = r.StartTime
sum.EndTime = r.EndTime
sum.Duration = r.EndTime.Unix() - r.EndTime.Unix()
sum.ScanStatus = job.ErrorStatus.String()
if job.Status(r.Status).Code() != -1 {
sum.ScanStatus = r.Status
}
// If the status is not success/stopped, there will not be any report.
if r.Status != job.SuccessStatus.String() &&
r.Status != job.StoppedStatus.String() {
return sum, nil
}
// Probably no report data if the job is interrupted
if len(r.Report) == 0 {
return nil, errors.Errorf("no report data for %s, status is: %s", r.UUID, sum.ScanStatus)
}
raw, err := ResolveData(r.MimeType, []byte(r.Report))
if err != nil {
return nil, err
}
rp, ok := raw.(*vuln.Report)
if !ok {
return nil, errors.Errorf("type mismatch: expect *vuln.Report but got %s", reflect.TypeOf(raw).String())
}
sum.Severity = rp.Severity
vsum := &vuln.VulnerabilitySummary{
Total: len(rp.Vulnerabilities),
Summary: make(vuln.SeveritySummary),
}
for _, v := range rp.Vulnerabilities {
if num, ok := vsum.Summary[v.Severity]; ok {
vsum.Summary[v.Severity] = num + 1
} else {
vsum.Summary[v.Severity] = 1
}
}
sum.Summary = vsum
return sum, nil
}

1
src/pkg/scan/report/supported_mime_test.go
View File

@ -69,6 +69,7 @@ func (suite *SupportedMimesSuite) SetupSuite() {
func (suite *SupportedMimesSuite) TestResolveData() {
obj, err := ResolveData(v1.MimeTypeNativeReport, suite.mockData)
require.NoError(suite.T(), err)
require.NotNil(suite.T(), obj)
require.Condition(suite.T(), func() (success bool) {
rp, ok := obj.(*vuln.Report)
success = ok && rp != nil && rp.Severity == vuln.High

12
src/pkg/scan/report/supported_mimes.go
View File

@ -31,13 +31,15 @@ var SupportedMimes = map[string]interface{}{
// ResolveData is a helper func to parse the JSON data with the given mime type.
func ResolveData(mime string, jsonData []byte) (interface{}, error) {
if len(jsonData) == 0 {
return nil, errors.New("empty JSON data")
}
// If no resolver defined for the given mime types, directly ignore it.
// The raw data will be used.
t, ok := SupportedMimes[mime]
if !ok {
return nil, errors.Errorf("report with mime type %s is not supported", mime)
return nil, nil
}
if len(jsonData) == 0 {
return nil, errors.New("empty JSON data")
}
ty := reflect.TypeOf(t)

23
src/pkg/scan/rest/v1/client.go
View File

@ -18,6 +18,7 @@ import (
"bytes"
"crypto/tls"
"encoding/json"
"fmt"
"io/ioutil"
"net"
"net/http"
@ -157,7 +158,7 @@ func (c *basicClient) SubmitScan(req *ScanRequest) (*ScanResponse, error) {
return nil, errors.Wrap(err, "v1 client: submit scan")
}
respData, err := c.send(request, generalResponseHandler(http.StatusCreated))
respData, err := c.send(request, generalResponseHandler(http.StatusAccepted))
if err != nil {
return nil, errors.Wrap(err, "v1 client: submit scan")
}
@ -199,7 +200,7 @@ func (c *basicClient) GetScanReport(scanRequestID, reportMIMEType string) (strin
func (c *basicClient) send(req *http.Request, h responseHandler) ([]byte, error) {
if c.authorizer != nil {
if err := c.authorizer.Authorize(req); err != nil {
return nil, errors.Wrap(err, "authorization")
return nil, errors.Wrap(err, "send: authorization")
}
}
@ -266,12 +267,24 @@ func generalRespHandlerFunc(expectedCode, code int, resp *http.Response) ([]byte
eResp := &ErrorResponse{
Err: &Error{},
}
if err := json.Unmarshal(buf, eResp); err == nil {
return nil, eResp
err := json.Unmarshal(buf, eResp)
if err != nil {
return nil, errors.Wrap(err, "general response handler")
}
// Append more contexts
eResp.Err.Message = fmt.Sprintf(
"%s: general response handler: unexpected status code: %d, expected: %d",
eResp.Err.Message,
code,
expectedCode,
)
return nil, eResp
}
return nil, errors.Errorf("unexpected status code: %d, response: %s", code, string(buf))
return nil, errors.Errorf("general response handler: unexpected status code: %d, expected: %d", code, expectedCode)
}
return buf, nil

5
src/pkg/scan/rest/v1/client_pool.go
View File

@ -102,7 +102,7 @@ func (bcp *basicClientPool) Get(r *scanner.Registration) (Client, error) {
return nil, errors.New("nil scanner registration")
}
if err := r.Validate(true); err != nil {
if err := r.Validate(false); err != nil {
return nil, errors.Wrap(err, "client pool: get")
}
@ -159,8 +159,7 @@ func (bcp *basicClientPool) deadCheck(key string, item *poolItem) {
}
func key(r *scanner.Registration) string {
return fmt.Sprintf("%s:%s:%s:%s:%v",
r.UUID,
return fmt.Sprintf("%s:%s:%s:%v",
r.URL,
r.Auth,
r.AccessCredential,

16
src/pkg/scan/rest/v1/client_test.go
View File

@ -115,7 +115,7 @@ type mockHandler struct{}
// ServeHTTP ...
func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
switch r.RequestURI {
case "/metadata":
case "/api/v1/metadata":
if r.Method != http.MethodGet {
w.WriteHeader(http.StatusForbidden)
return
@ -126,7 +126,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
Vendor: "Harbor",
Version: "0.1.0",
},
Capabilities: &ScannerCapability{
Capabilities: []*ScannerCapability{{
ConsumesMimeTypes: []string{
MimeTypeOCIArtifact,
MimeTypeDockerArtifact,
@ -135,7 +135,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
MimeTypeNativeReport,
MimeTypeRawReport,
},
},
}},
Properties: ScannerProperties{
"extra": "testing",
},
@ -144,7 +144,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write(data)
break
case "/scan":
case "/api/v1/scan":
if r.Method != http.MethodPost {
w.WriteHeader(http.StatusForbidden)
return
@ -155,10 +155,10 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
data, _ := json.Marshal(res)
w.WriteHeader(http.StatusCreated)
w.WriteHeader(http.StatusAccepted)
_, _ = w.Write(data)
break
case "/scan/id1/report":
case "/api/v1/scan/id1/report":
if r.Method != http.MethodGet {
w.WriteHeader(http.StatusForbidden)
return
@ -175,7 +175,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write(data)
break
case "/scan/id2/report":
case "/api/v1/scan/id2/report":
if r.Method != http.MethodGet {
w.WriteHeader(http.StatusForbidden)
return
@ -183,7 +183,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("{}"))
break
case "/scan/id3/report":
case "/api/v1/scan/id3/report":
if r.Method != http.MethodGet {
w.WriteHeader(http.StatusForbidden)
return

11
src/pkg/scan/rest/v1/models.go
View File

@ -60,16 +60,21 @@ type ScannerProperties map[string]string
// a scanner capable of scanning a given Artifact stored in its registry and making sure that it
// can interpret a returned result.
type ScannerAdapterMetadata struct {
Scanner *Scanner `json:"scanner"`
Capabilities *ScannerCapability `json:"capabilities"`
Properties ScannerProperties `json:"properties"`
Scanner *Scanner `json:"scanner"`
Capabilities []*ScannerCapability `json:"capabilities"`
Properties ScannerProperties `json:"properties"`
}
// Artifact represents an artifact stored in Registry.
type Artifact struct {
// ID of the namespace (project). It will not be sent to scanner adapter.
NamespaceID int64 `json:"-"`
// The full name of a Harbor repository containing the artifact, including the namespace.
// For example, `library/oracle/nosql`.
Repository string `json:"repository"`
// The info used to identify the version of the artifact,
// e.g: tag of image or version of the chart.
Tag string `json:"tag"`
// The artifact's digest, consisting of an algorithm and hex portion.
// For example, `sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b`,
// represents sha256 based digest.

4
src/pkg/scan/rest/v1/spec.go
View File

@ -39,6 +39,8 @@ const (
MimeTypeScanRequest = "application/vnd.scanner.adapter.scan.request+json; version=1.0"
// MimeTypeScanResponse defines the mime type for scan response
MimeTypeScanResponse = "application/vnd.scanner.adapter.scan.response+json; version=1.0"
apiPrefix = "/api/v1"
)
// RequestResolver is a function template to modify the API request, e.g: add headers
@ -70,6 +72,8 @@ func NewSpec(base string) *Spec {
}
}
s.baseRoute = fmt.Sprintf("%s%s", s.baseRoute, apiPrefix)
return s
}

2
src/pkg/scan/vuln/report.go
View File

@ -54,5 +54,5 @@ type VulnerabilityItem struct {
// The list of link to the upstream database with the full description of the vulnerability.
// Format: URI
// e.g: List [ "https://security-tracker.debian.org/tracker/CVE-2017-8283" ]
Links []string
Links []string `json:"links"`
}

41
src/pkg/scan/vuln/summary.go Normal file
View File

@ -0,0 +1,41 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package vuln
import (
"time"
)
// NativeReportSummary is the default supported scan report summary model.
// Generated based on the report with v1.MimeTypeNativeReport mime type.
type NativeReportSummary struct {
ReportID string `json:"report_id"`
ScanStatus string `json:"scan_status"`
Severity Severity `json:"severity"`
Duration int64 `json:"duration"`
Summary *VulnerabilitySummary `json:"summary"`
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
}
// VulnerabilitySummary contains the total number of the found vulnerabilities number
// and numbers of each severity level.
type VulnerabilitySummary struct {
Total int `json:"total"`
Summary SeveritySummary `json:"summary"`
}
// SeveritySummary ...
type SeveritySummary map[Severity]int

214
src/replication/adapter/quayio/adapter.go Normal file
View File

@ -0,0 +1,214 @@
package quayio
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"strings"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
type adapter struct {
*native.Adapter
registry *model.Registry
client *common_http.Client
}
func init() {
err := adp.RegisterFactory(model.RegistryTypeQuayio, func(registry *model.Registry) (adp.Adapter, error) {
return newAdapter(registry)
})
if err != nil {
log.Errorf("failed to register factory for Quay.io: %v", err)
return
}
log.Infof("the factory of Quay.io adapter was registered")
}
func newAdapter(registry *model.Registry) (*adapter, error) {
modifiers := []modifier.Modifier{
&auth.UserAgentModifier{
UserAgent: adp.UserAgentReplication,
},
}
var authorizer modifier.Modifier
if registry.Credential != nil && len(registry.Credential.AccessKey) != 0 {
authorizer = auth.NewAPIKeyAuthorizer("Authorization", fmt.Sprintf("Bearer %s", registry.Credential.AccessKey), auth.APIKeyInHeader)
}
if authorizer != nil {
modifiers = append(modifiers, authorizer)
}
nativeRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
if err != nil {
return nil, err
}
return &adapter{
Adapter: nativeRegistryAdapter,
registry: registry,
client: common_http.NewClient(
&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
},
modifiers...,
),
}, nil
}
// Info returns information of the registry
func (a *adapter) Info() (*model.RegistryInfo, error) {
return &model.RegistryInfo{
Type: model.RegistryTypeQuayio,
SupportedResourceTypes: []model.ResourceType{
model.ResourceTypeImage,
},
SupportedResourceFilters: []*model.FilterStyle{
{
Type: model.FilterTypeName,
Style: model.FilterStyleTypeText,
},
{
Type: model.FilterTypeTag,
Style: model.FilterStyleTypeText,
},
},
SupportedTriggers: []model.TriggerType{
model.TriggerTypeManual,
model.TriggerTypeScheduled,
},
}, nil
}
// HealthCheck checks health status of a registry
func (a *adapter) HealthCheck() (model.HealthStatus, error) {
err := a.PingSimple()
if err != nil {
return model.Unhealthy, nil
}
return model.Healthy, nil
}
// PrepareForPush does the prepare work that needed for pushing/uploading the resource
// eg: create the namespace or repository
func (a *adapter) PrepareForPush(resources []*model.Resource) error {
namespaces := []string{}
for _, resource := range resources {
if resource == nil {
return errors.New("the resource cannot be null")
}
if resource.Metadata == nil {
return errors.New("the metadata of resource cannot be null")
}
if resource.Metadata.Repository == nil {
return errors.New("the namespace of resource cannot be null")
}
if len(resource.Metadata.Repository.Name) == 0 {
return errors.New("the name of the namespace cannot be null")
}
paths := strings.Split(resource.Metadata.Repository.Name, "/")
namespace := paths[0]
namespaces = append(namespaces, namespace)
}
for _, namespace := range namespaces {
err := a.createNamespace(&model.Namespace{
Name: namespace,
})
if err != nil {
return fmt.Errorf("create namespace '%s' in Quay.io error: %v", namespace, err)
}
log.Debugf("namespace %s created", namespace)
}
return nil
}
// createNamespace creates a new namespace in Quay.io
func (a *adapter) createNamespace(namespace *model.Namespace) error {
ns, err := a.getNamespace(namespace.Name)
if err != nil {
return fmt.Errorf("check existence of namespace '%s' error: %v", namespace.Name, err)
}
// If the namespace already exist, return succeeded directly.
if ns != nil {
log.Infof("Namespace %s already exist in Quay.io, skip it.", namespace.Name)
return nil
}
org := &orgCreate{
Name: namespace.Name,
Email: namespace.GetStringMetadata("email", namespace.Name),
}
b, err := json.Marshal(org)
if err != nil {
return err
}
req, err := http.NewRequest(http.MethodPost, buildOrgURL(""), bytes.NewReader(b))
if err != nil {
return err
}
resp, err := a.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode/100 == 2 {
return nil
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
log.Errorf("create namespace error: %d -- %s", resp.StatusCode, string(body))
return fmt.Errorf("%d -- %s", resp.StatusCode, body)
}
// getNamespace get namespace from Quay.io, if the namespace not found, two nil would be returned.
func (a *adapter) getNamespace(namespace string) (*model.Namespace, error) {
req, err := http.NewRequest(http.MethodGet, buildOrgURL(namespace), nil)
if err != nil {
return nil, err
}
resp, err := a.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode == http.StatusNotFound {
return nil, nil
}
if resp.StatusCode/100 != 2 {
log.Errorf("get namespace error: %d -- %s", resp.StatusCode, string(body))
return nil, fmt.Errorf("%d -- %s", resp.StatusCode, body)
}
return &model.Namespace{
Name: namespace,
}, nil
}

48
src/replication/adapter/quayio/adapter_test.go Normal file
View File

@ -0,0 +1,48 @@
package quayio
import (
"testing"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/stretchr/testify/assert"
)
func getMockAdapter(t *testing.T) adp.Adapter {
factory, _ := adp.GetFactory(model.RegistryTypeQuayio)
adapter, err := factory(&model.Registry{
Type: model.RegistryTypeQuayio,
URL: "https://quay.io",
})
assert.Nil(t, err)
return adapter
}
func TestAdapter_NewAdapter(t *testing.T) {
factory, err := adp.GetFactory("BadName")
assert.Nil(t, factory)
assert.NotNil(t, err)
factory, err = adp.GetFactory(model.RegistryTypeQuayio)
assert.Nil(t, err)
assert.NotNil(t, factory)
}
func TestAdapter_HealthCheck(t *testing.T) {
health, err := getMockAdapter(t).HealthCheck()
assert.Nil(t, err)
assert.Equal(t, string(health), model.Healthy)
}
func TestAdapter_Info(t *testing.T) {
info, err := getMockAdapter(t).Info()
assert.Nil(t, err)
t.Log(info)
}
func TestAdapter_PullManifests(t *testing.T) {
quayAdapter := getMockAdapter(t)
registry, _, err := quayAdapter.(*adapter).PullManifest("quay/busybox", "latest", []string{})
assert.Nil(t, err)
assert.NotNil(t, registry)
t.Log(registry)
}

12
src/replication/adapter/quayio/types.go Normal file
View File

@ -0,0 +1,12 @@
package quayio
import "fmt"
type orgCreate struct {
Name string `json:"name"`
Email string `json:"email"`
}
func buildOrgURL(orgName string) string {
return fmt.Sprintf("https://quay.io/api/v1/organization/%s", orgName)
}

1
src/replication/model/registry.go
View File

@ -31,6 +31,7 @@ const (
RegistryTypeAzureAcr RegistryType = "azure-acr"
RegistryTypeAliAcr RegistryType = "ali-acr"
RegistryTypeJfrogArtifactory RegistryType = "jfrog-artifactory"
RegistryTypeQuayio RegistryType = "quay-io"
RegistryTypeHelmHub RegistryType = "helm-hub"

2
src/replication/replication.go
View File

@ -45,6 +45,8 @@ import (
_ "github.com/goharbor/harbor/src/replication/adapter/aliacr"
// register the Jfrog Artifactory adapter
_ "github.com/goharbor/harbor/src/replication/adapter/jfrog"
// register the Quay.io adapter
_ "github.com/goharbor/harbor/src/replication/adapter/quayio"
// register the Helm Hub adapter
_ "github.com/goharbor/harbor/src/replication/adapter/helmhub"
)

10
tests/robot-cases/Group0-BAT/API_DB.robot
View File

@ -29,16 +29,18 @@ Test Case - Add Replication Rule
Harbor API Test ./tests/apitests/python/test_add_replication_rule.py
Test Case - Edit Project Creation
Harbor API Test ./tests/apitests/python/test_edit_project_creation.py
Test Case - Scan Image
Harbor API Test ./tests/apitests/python/test_scan_image.py
*** Enable this case after deployment change PR merged ***
*** Test Case - Scan Image ***
*** Harbor API Test ./tests/apitests/python/test_scan_image.py ***
Test Case - Manage Project Member
Harbor API Test ./tests/apitests/python/test_manage_project_member.py
Test Case - Project Level Policy Content Trust
Harbor API Test ./tests/apitests/python/test_project_level_policy_content_trust.py
Test Case - User View Logs
Harbor API Test ./tests/apitests/python/test_user_view_logs.py
Test Case - Scan All Images
Harbor API Test ./tests/apitests/python/test_scan_all_images.py
*** Enable this case after deployment change PR merged ***
*** Test Case - Scan All Images ***
*** Harbor API Test ./tests/apitests/python/test_scan_all_images.py ***
Test Case - List Helm Charts
Harbor API Test ./tests/apitests/python/test_list_helm_charts.py
Test Case - Assign Sys Admin