diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 191fe4775e..e2a83c8aba 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -959,13 +959,13 @@ paths: description: User ID does not exist. '500': description: Unexpected internal errors. - '/users/{user_id}/gen_cli_secret': - post: - summary: Generate new CLI secret for a user. + '/users/{user_id}/cli_secret': + put: + summary: Set CLI secret for a user. description: | This endpoint let user generate a new CLI secret for himself. This API only works when auth mode is set to 'OIDC'. Once this API returns with successful status, the old secret will be invalid, as there will be only one CLI secret - for a user. The new secret will be returned in the response. + for a user. parameters: - name: user_id in: path @@ -973,19 +973,23 @@ paths: format: int required: true description: User ID - tags: - - Products - responses: - '200': - description: The secret is successfully generated. + - name: input_secret + in: body + description: JSON object that includes the new secret + required: true schema: type: object properties: secret: type: string description: The new secret + tags: + - Products + responses: + '200': + description: The secret is successfully updated '400': - description: Invalid user ID. Or user is not onboarded via OIDC authentication. + description: Invalid user ID. Or user is not onboarded via OIDC authentication. Or the secret does not meet the standard. '401': description: User need to log in first. '403': diff --git a/docs/user_guide.md b/docs/user_guide.md index 5832ab13b5..42d08e7db5 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -906,7 +906,7 @@ For example, you have following tags, listed according to their push time, and a You configure a retention policy to retain the two latest tags that match `harbor-*`, so that `harbor-rc` and `harbor-latest` are deleted. However, since all tags refer to the same SHA digest, this policy would also delete the tags `harbor-1.8` and `harbor-release`, so all tags are retained. -### Combining Rules on a Respository +### Combining Rules on a Repository You can define up to 15 rules per project. You can apply multiple rules to a repository or set of repositories. When you apply multiple rules to a repository, they are applied with `OR` logic rather than with `AND` logic. In this way, there is no prioritization of application of the rules on a given repository. Rules run concurrently in the background, and the resulting sets from each rule are combined at the end of the run. diff --git a/make/migrations/postgresql/0012_1.10.0_schema.up.sql b/make/migrations/postgresql/0012_1.10.0_schema.up.sql deleted file mode 100644 index 57f6ff4ff2..0000000000 --- a/make/migrations/postgresql/0012_1.10.0_schema.up.sql +++ /dev/null @@ -1,34 +0,0 @@ -/*Table for keeping the plug scanner registration*/ -CREATE TABLE scanner_registration -( - id SERIAL PRIMARY KEY NOT NULL, - uuid VARCHAR(64) UNIQUE NOT NULL, - url VARCHAR(256) UNIQUE NOT NULL, - name VARCHAR(128) UNIQUE NOT NULL, - description VARCHAR(1024) NULL, - auth VARCHAR(16) NOT NULL, - access_cred VARCHAR(512) NULL, - disabled BOOLEAN NOT NULL DEFAULT FALSE, - is_default BOOLEAN NOT NULL DEFAULT FALSE, - skip_cert_verify BOOLEAN NOT NULL DEFAULT FALSE, - create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); - -/*Table for keeping the scan report. The report details are stored as JSON*/ -CREATE TABLE scan_report -( - id SERIAL PRIMARY KEY NOT NULL, - uuid VARCHAR(64) UNIQUE NOT NULL, - digest VARCHAR(256) NOT NULL, - registration_uuid VARCHAR(64) NOT NULL, - mime_type VARCHAR(256) NOT NULL, - job_id VARCHAR(32), - status VARCHAR(16) NOT NULL, - status_code INTEGER DEFAULT 0, - status_rev BIGINT DEFAULT 0, - report JSON, - start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - end_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(digest, registration_uuid, mime_type) -) diff --git a/make/migrations/postgresql/0015_1.10.0_schema.up.sql b/make/migrations/postgresql/0015_1.10.0_schema.up.sql index beedeb3163..44372ba938 100644 --- a/make/migrations/postgresql/0015_1.10.0_schema.up.sql +++ b/make/migrations/postgresql/0015_1.10.0_schema.up.sql @@ -1,3 +1,39 @@ +/*Table for keeping the plug scanner registration*/ +CREATE TABLE scanner_registration +( + id SERIAL PRIMARY KEY NOT NULL, + uuid VARCHAR(64) UNIQUE NOT NULL, + url VARCHAR(256) UNIQUE NOT NULL, + name VARCHAR(128) UNIQUE NOT NULL, + description VARCHAR(1024) NULL, + auth VARCHAR(16) NOT NULL, + access_cred VARCHAR(512) NULL, + disabled BOOLEAN NOT NULL DEFAULT FALSE, + is_default BOOLEAN NOT NULL DEFAULT FALSE, + skip_cert_verify BOOLEAN NOT NULL DEFAULT FALSE, + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +/*Table for keeping the scan report. The report details are stored as JSON*/ +CREATE TABLE scan_report +( + id SERIAL PRIMARY KEY NOT NULL, + uuid VARCHAR(64) UNIQUE NOT NULL, + digest VARCHAR(256) NOT NULL, + registration_uuid VARCHAR(64) NOT NULL, + mime_type VARCHAR(256) NOT NULL, + job_id VARCHAR(64), + track_id VARCHAR(64), + status VARCHAR(1024) NOT NULL, + status_code INTEGER DEFAULT 0, + status_rev BIGINT DEFAULT 0, + report JSON, + start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + end_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(digest, registration_uuid, mime_type) +); + /** Add table for immutable tag **/ CREATE TABLE immutable_tag_rule ( diff --git a/src/common/models/repo.go b/src/common/models/repo.go index 9993fbcc60..6562e95312 100644 --- a/src/common/models/repo.go +++ b/src/common/models/repo.go @@ -54,11 +54,11 @@ type RepositoryQuery struct { // TagResp holds the information of one image tag type TagResp struct { TagDetail - Signature *model.Target `json:"signature"` - ScanOverview *ImgScanOverview `json:"scan_overview,omitempty"` - Labels []*Label `json:"labels"` - PushTime time.Time `json:"push_time"` - PullTime time.Time `json:"pull_time"` + Signature *model.Target `json:"signature"` + ScanOverview map[string]interface{} `json:"scan_overview,omitempty"` + Labels []*Label `json:"labels"` + PushTime time.Time `json:"push_time"` + PullTime time.Time `json:"pull_time"` } // TagDetail ... diff --git a/src/common/rbac/const.go b/src/common/rbac/const.go index aa549116aa..6b850b6e96 100755 --- a/src/common/rbac/const.go +++ b/src/common/rbac/const.go @@ -54,9 +54,10 @@ const ( ResourceRepositoryTag = Resource("repository-tag") ResourceRepositoryTagLabel = Resource("repository-tag-label") ResourceRepositoryTagManifest = Resource("repository-tag-manifest") - ResourceRepositoryTagScanJob = Resource("repository-tag-scan-job") - ResourceRepositoryTagVulnerability = Resource("repository-tag-vulnerability") + ResourceRepositoryTagScanJob = Resource("repository-tag-scan-job") // TODO: remove + ResourceRepositoryTagVulnerability = Resource("repository-tag-vulnerability") // TODO: remove ResourceRobot = Resource("robot") ResourceNotificationPolicy = Resource("notification-policy") + ResourceScan = Resource("scan") ResourceSelf = Resource("") // subresource for self ) diff --git a/src/common/rbac/project/util.go b/src/common/rbac/project/util.go index ce69800aad..85116fe21b 100644 --- a/src/common/rbac/project/util.go +++ b/src/common/rbac/project/util.go @@ -162,6 +162,9 @@ var ( {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionDelete}, {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList}, {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionRead}, + + {Resource: rbac.ResourceScan, Action: rbac.ActionCreate}, + {Resource: rbac.ResourceScan, Action: rbac.ActionRead}, } ) diff --git a/src/common/rbac/project/visitor_role.go b/src/common/rbac/project/visitor_role.go index 651252cdb7..d8d594f4ff 100755 --- a/src/common/rbac/project/visitor_role.go +++ b/src/common/rbac/project/visitor_role.go @@ -119,6 +119,9 @@ var ( {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionDelete}, {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList}, {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionRead}, + + {Resource: rbac.ResourceScan, Action: rbac.ActionCreate}, + {Resource: rbac.ResourceScan, Action: rbac.ActionRead}, }, "master": { @@ -201,6 +204,9 @@ var ( {Resource: rbac.ResourceRobot, Action: rbac.ActionList}, {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList}, + + {Resource: rbac.ResourceScan, Action: rbac.ActionCreate}, + {Resource: rbac.ResourceScan, Action: rbac.ActionRead}, }, "developer": { @@ -251,6 +257,9 @@ var ( {Resource: rbac.ResourceRobot, Action: rbac.ActionRead}, {Resource: rbac.ResourceRobot, Action: rbac.ActionList}, + + {Resource: rbac.ResourceScan, Action: rbac.ActionCreate}, + {Resource: rbac.ResourceScan, Action: rbac.ActionRead}, }, "guest": { diff --git a/src/common/utils/registry/auth/apikey.go b/src/common/utils/registry/auth/apikey.go new file mode 100644 index 0000000000..1dd02b16e0 --- /dev/null +++ b/src/common/utils/registry/auth/apikey.go @@ -0,0 +1,45 @@ +package auth + +import ( + "fmt" + "net/http" + + "github.com/goharbor/harbor/src/common/http/modifier" +) + +type apiKeyType = string + +const ( + // APIKeyInHeader sets auth content in header + APIKeyInHeader apiKeyType = "header" + // APIKeyInQuery sets auth content in url query + APIKeyInQuery apiKeyType = "query" +) + +type apiKeyAuthorizer struct { + key, value, in apiKeyType +} + +// NewAPIKeyAuthorizer returns a apikey authorizer +func NewAPIKeyAuthorizer(key, value, in apiKeyType) modifier.Modifier { + return &apiKeyAuthorizer{ + key: key, + value: value, + in: in, + } +} + +// Modify implements modifier.Modifier +func (a *apiKeyAuthorizer) Modify(r *http.Request) error { + switch a.in { + case APIKeyInHeader: + r.Header.Set(a.key, a.value) + return nil + case APIKeyInQuery: + query := r.URL.Query() + query.Add(a.key, a.value) + r.URL.RawQuery = query.Encode() + return nil + } + return fmt.Errorf("set api key in %s is invalid", a.in) +} diff --git a/src/common/utils/registry/auth/apikey_test.go b/src/common/utils/registry/auth/apikey_test.go new file mode 100644 index 0000000000..ff6ef41336 --- /dev/null +++ b/src/common/utils/registry/auth/apikey_test.go @@ -0,0 +1,50 @@ +package auth + +import ( + "net/http" + "testing" + + "github.com/goharbor/harbor/src/common/http/modifier" + "github.com/stretchr/testify/assert" +) + +func TestAPIKeyAuthorizer(t *testing.T) { + type suite struct { + key string + value string + in string + } + + var ( + s suite + authorizer modifier.Modifier + request *http.Request + err error + ) + + // set in header + s = suite{key: "Authorization", value: "Basic abc", in: "header"} + authorizer = NewAPIKeyAuthorizer(s.key, s.value, s.in) + request, err = http.NewRequest(http.MethodGet, "http://example.com", nil) + assert.Nil(t, err) + err = authorizer.Modify(request) + assert.Nil(t, err) + assert.Equal(t, s.value, request.Header.Get(s.key)) + + // set in query + s = suite{key: "private_token", value: "abc", in: "query"} + authorizer = NewAPIKeyAuthorizer(s.key, s.value, s.in) + request, err = http.NewRequest(http.MethodGet, "http://example.com", nil) + assert.Nil(t, err) + err = authorizer.Modify(request) + assert.Nil(t, err) + assert.Equal(t, s.value, request.URL.Query().Get(s.key)) + + // set in invalid location + s = suite{key: "", value: "", in: "invalid"} + authorizer = NewAPIKeyAuthorizer(s.key, s.value, s.in) + request, err = http.NewRequest(http.MethodGet, "http://example.com", nil) + assert.Nil(t, err) + err = authorizer.Modify(request) + assert.NotNil(t, err) +} diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 7cab5f769d..f60b500b49 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -211,8 +211,17 @@ func init() { scannerAPI := &ScannerAPI{} beego.Router("/api/scanners", scannerAPI, "post:Create;get:List") beego.Router("/api/scanners/:uuid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault") + beego.Router("/api/scanners/:uuid/metadata", scannerAPI, "get:Metadata") + beego.Router("/api/scanners/ping", scannerAPI, "post:Ping") + // Add routes for project level scanner - beego.Router("/api/projects/:pid([0-9]+)/scanner", scannerAPI, "get:GetProjectScanner;put:SetProjectScanner") + proScannerAPI := &ProjectScannerAPI{} + beego.Router("/api/projects/:pid([0-9]+)/scanner", proScannerAPI, "get:GetProjectScanner;put:SetProjectScanner") + + // Add routes for scan + scanAPI := &ScanAPI{} + beego.Router("/api/repositories/*/tags/:tag/scan", scanAPI, "post:Scan;get:Report") + beego.Router("/api/repositories/*/tags/:tag/scan/:uuid/log", scanAPI, "get:Log") // syncRegistry if err := SyncRegistry(config.GlobalProjectMgr); err != nil { diff --git a/src/core/api/pro_scanner.go b/src/core/api/pro_scanner.go new file mode 100644 index 0000000000..ff0e45436b --- /dev/null +++ b/src/core/api/pro_scanner.go @@ -0,0 +1,112 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "github.com/goharbor/harbor/src/common/rbac" + "github.com/goharbor/harbor/src/pkg/scan/api/scanner" + "github.com/pkg/errors" +) + +// ProjectScannerAPI provides rest API for managing the project level scanner(s). +type ProjectScannerAPI struct { + // The base controller to provide common utilities + BaseController + // Scanner controller for operating scanner registrations. + c scanner.Controller + // ID of the project + pid int64 +} + +// Prepare sth. for the subsequent actions +func (sa *ProjectScannerAPI) Prepare() { + // Call super prepare method + sa.BaseController.Prepare() + + // Check access permissions + if !sa.RequireAuthenticated() { + return + } + + // Get ID of the project + pid, err := sa.GetInt64FromPath(":pid") + if err != nil { + sa.SendBadRequestError(errors.Wrap(err, "project scanner API")) + return + } + + // Check if the project exists + exists, err := sa.ProjectMgr.Exists(pid) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "project scanner API")) + return + } + + if !exists { + sa.SendNotFoundError(errors.Errorf("project with id %d", sa.pid)) + return + } + + sa.pid = pid + + sa.c = scanner.DefaultController +} + +// GetProjectScanner gets the project level scanner +func (sa *ProjectScannerAPI) GetProjectScanner() { + // Check access permissions + if !sa.RequireProjectAccess(sa.pid, rbac.ActionRead, rbac.ResourceConfiguration) { + return + } + + r, err := sa.c.GetRegistrationByProject(sa.pid) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: get project scanners")) + return + } + + if r != nil { + sa.Data["json"] = r + } else { + sa.Data["json"] = make(map[string]interface{}) + } + + sa.ServeJSON() +} + +// SetProjectScanner sets the project level scanner +func (sa *ProjectScannerAPI) SetProjectScanner() { + // Check access permissions + if !sa.RequireProjectAccess(sa.pid, rbac.ActionUpdate, rbac.ResourceConfiguration) { + return + } + + body := make(map[string]string) + if err := sa.DecodeJSONReq(&body); err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: set project scanners")) + return + } + + uuid, ok := body["uuid"] + if !ok || len(uuid) == 0 { + sa.SendBadRequestError(errors.New("missing scanner uuid when setting project scanner")) + return + } + + if err := sa.c.SetRegistrationByProject(sa.pid, uuid); err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: set project scanners")) + return + } +} diff --git a/src/core/api/pro_scanner_test.go b/src/core/api/pro_scanner_test.go new file mode 100644 index 0000000000..42d64f305d --- /dev/null +++ b/src/core/api/pro_scanner_test.go @@ -0,0 +1,95 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "fmt" + "net/http" + "testing" + + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner" + "github.com/stretchr/testify/suite" +) + +// ProScannerAPITestSuite is test suite for testing the project scanner API +type ProScannerAPITestSuite struct { + suite.Suite + + originC sc.Controller + mockC *MockScannerAPIController +} + +// TestProScannerAPI is the entry of ProScannerAPITestSuite +func TestProScannerAPI(t *testing.T) { + suite.Run(t, new(ProScannerAPITestSuite)) +} + +// SetupSuite prepares testing env +func (suite *ProScannerAPITestSuite) SetupTest() { + suite.originC = sc.DefaultController + m := &MockScannerAPIController{} + sc.DefaultController = m + + suite.mockC = m +} + +// TearDownTest clears test case env +func (suite *ProScannerAPITestSuite) TearDownTest() { + // Restore + sc.DefaultController = suite.originC +} + +// TestScannerAPIProjectScanner tests the API of getting/setting project level scanner +func (suite *ProScannerAPITestSuite) TestScannerAPIProjectScanner() { + suite.mockC.On("SetRegistrationByProject", int64(1), "uuid").Return(nil) + + // Set + body := make(map[string]interface{}, 1) + body["uuid"] = "uuid" + runCodeCheckingCases(suite.T(), &codeCheckingCase{ + request: &testingRequest{ + url: fmt.Sprintf("/api/projects/%d/scanner", 1), + method: http.MethodPut, + credential: projAdmin, + bodyJSON: body, + }, + code: http.StatusOK, + }) + + r := &scanner.Registration{ + ID: 1004, + UUID: "uuid", + Name: "TestScannerAPIProjectScanner", + Description: "JUST FOR TEST", + URL: "https://a.b.c", + } + suite.mockC.On("GetRegistrationByProject", int64(1)).Return(r, nil) + + // Get + rr := &scanner.Registration{} + err := handleAndParse(&testingRequest{ + url: fmt.Sprintf("/api/projects/%d/scanner", 1), + method: http.MethodGet, + credential: projAdmin, + }, rr) + require.NoError(suite.T(), err) + + assert.Equal(suite.T(), r.Name, rr.Name) + assert.Equal(suite.T(), r.UUID, rr.UUID) +} diff --git a/src/core/api/repository.go b/src/core/api/repository.go index 8a40d5f49e..8227a5caed 100755 --- a/src/core/api/repository.go +++ b/src/core/api/repository.go @@ -25,6 +25,11 @@ import ( "strings" "time" + "github.com/goharbor/harbor/src/jobservice/logger" + + "github.com/goharbor/harbor/src/pkg/scan/api/scan" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" "github.com/goharbor/harbor/src/common" @@ -40,7 +45,6 @@ import ( "github.com/goharbor/harbor/src/core/config" notifierEvt "github.com/goharbor/harbor/src/core/notifier/event" coreutils "github.com/goharbor/harbor/src/core/utils" - "github.com/goharbor/harbor/src/pkg/scan" "github.com/goharbor/harbor/src/replication" "github.com/goharbor/harbor/src/replication/event" "github.com/goharbor/harbor/src/replication/model" @@ -397,6 +401,13 @@ func (ra *RepositoryAPI) GetTag() { return } + project, err := ra.ProjectMgr.Get(projectName) + if err != nil { + ra.ParseAndHandleError(fmt.Sprintf("failed to get the project %s", + projectName), err) + return + } + client, err := coreutils.NewRepositoryClientForUI(ra.SecurityCtx.GetUsername(), repository) if err != nil { ra.SendInternalServerError(fmt.Errorf("failed to initialize the client for %s: %v", @@ -414,7 +425,7 @@ func (ra *RepositoryAPI) GetTag() { return } - result := assembleTagsInParallel(client, repository, []string{tag}, + result := assembleTagsInParallel(client, project.ProjectID, repository, []string{tag}, ra.SecurityCtx.GetUsername()) ra.Data["json"] = result[0] ra.ServeJSON() @@ -523,14 +534,14 @@ func (ra *RepositoryAPI) GetTags() { } projectName, _ := utils.ParseRepository(repoName) - exist, err := ra.ProjectMgr.Exists(projectName) + project, err := ra.ProjectMgr.Get(projectName) if err != nil { - ra.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s", + ra.ParseAndHandleError(fmt.Sprintf("failed to get the project %s", projectName), err) return } - if !exist { + if project == nil { ra.SendNotFoundError(fmt.Errorf("project %s not found", projectName)) return } @@ -587,8 +598,13 @@ func (ra *RepositoryAPI) GetTags() { return } - ra.Data["json"] = assembleTagsInParallel(client, repoName, tags, - ra.SecurityCtx.GetUsername()) + ra.Data["json"] = assembleTagsInParallel( + client, + project.ProjectID, + repoName, + tags, + ra.SecurityCtx.GetUsername(), + ) ra.ServeJSON() } @@ -607,7 +623,7 @@ func simpleTags(tags []string) []*models.TagResp { // get config, signature and scan overview and assemble them into one // struct for each tag in tags -func assembleTagsInParallel(client *registry.Repository, repository string, +func assembleTagsInParallel(client *registry.Repository, projectID int64, repository string, tags []string, username string) []*models.TagResp { var err error signatures := map[string][]notarymodel.Target{} @@ -621,8 +637,15 @@ func assembleTagsInParallel(client *registry.Repository, repository string, c := make(chan *models.TagResp) for _, tag := range tags { - go assembleTag(c, client, repository, tag, config.WithClair(), - config.WithNotary(), signatures) + go assembleTag( + c, + client, + projectID, + repository, + tag, + config.WithNotary(), + signatures, + ) } result := []*models.TagResp{} var item *models.TagResp @@ -636,8 +659,8 @@ func assembleTagsInParallel(client *registry.Repository, repository string, return result } -func assembleTag(c chan *models.TagResp, client *registry.Repository, - repository, tag string, clairEnabled, notaryEnabled bool, +func assembleTag(c chan *models.TagResp, client *registry.Repository, projectID int64, + repository, tag string, notaryEnabled bool, signatures map[string][]notarymodel.Target) { item := &models.TagResp{} // labels @@ -659,8 +682,9 @@ func assembleTag(c chan *models.TagResp, client *registry.Repository, } // scan overview - if clairEnabled { - item.ScanOverview = getScanOverview(item.Digest, item.Name) + so := getSummary(projectID, repository, item.Digest) + if len(so) > 0 { + item.ScanOverview = so } // signature, compare both digest and tag @@ -968,73 +992,6 @@ func (ra *RepositoryAPI) GetSignatures() { ra.ServeJSON() } -// ScanImage handles request POST /api/repository/$repository/tags/$tag/scan to trigger image scan manually. -func (ra *RepositoryAPI) ScanImage() { - if !config.WithClair() { - log.Warningf("Harbor is not deployed with Clair, scan is disabled.") - ra.SendInternalServerError(errors.New("harbor is not deployed with Clair, scan is disabled")) - return - } - repoName := ra.GetString(":splat") - tag := ra.GetString(":tag") - projectName, _ := utils.ParseRepository(repoName) - exist, err := ra.ProjectMgr.Exists(projectName) - if err != nil { - ra.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s", - projectName), err) - return - } - if !exist { - ra.SendNotFoundError(fmt.Errorf("project %s not found", projectName)) - return - } - if !ra.SecurityCtx.IsAuthenticated() { - ra.SendUnAuthorizedError(errors.New("Unauthorized")) - return - } - - if !ra.RequireProjectAccess(projectName, rbac.ActionCreate, rbac.ResourceRepositoryTagScanJob) { - return - } - err = coreutils.TriggerImageScan(repoName, tag) - if err != nil { - log.Errorf("Error while calling job service to trigger image scan: %v", err) - ra.SendInternalServerError(errors.New("Failed to scan image, please check log for details")) - return - } -} - -// VulnerabilityDetails fetch vulnerability info from clair, transform to Harbor's format and return to client. -func (ra *RepositoryAPI) VulnerabilityDetails() { - if !config.WithClair() { - log.Warningf("Harbor is not deployed with Clair, it's not impossible to get vulnerability details.") - ra.SendInternalServerError(errors.New("harbor is not deployed with Clair, it's not impossible to get vulnerability details")) - return - } - repository := ra.GetString(":splat") - tag := ra.GetString(":tag") - exist, digest, err := ra.checkExistence(repository, tag) - if err != nil { - ra.SendInternalServerError(fmt.Errorf("failed to check the existence of resource, error: %v", err)) - return - } - if !exist { - ra.SendNotFoundError(fmt.Errorf("resource: %s:%s not found", repository, tag)) - return - } - - projectName, _ := utils.ParseRepository(repository) - if !ra.RequireProjectAccess(projectName, rbac.ActionList, rbac.ResourceRepositoryTagVulnerability) { - return - } - res, err := scan.VulnListByDigest(digest) - if err != nil { - log.Errorf("Failed to get vulnerability list for image: %s:%s", repository, tag) - } - ra.Data["json"] = res - ra.ServeJSON() -} - func getSignatures(username, repository string) (map[string][]notarymodel.Target, error) { targets, err := notary.GetInternalTargets(config.InternalNotaryEndpoint(), username, repository) @@ -1079,33 +1036,19 @@ func (ra *RepositoryAPI) checkExistence(repository, tag string) (bool, string, e return true, digest, nil } -// will return nil when it failed to get data. The parm "tag" is for logging only. -func getScanOverview(digest string, tag string) *models.ImgScanOverview { - if len(digest) == 0 { - log.Debug("digest is nil") - return nil +func getSummary(pid int64, repository string, digest string) map[string]interface{} { + // At present, only get harbor native report as default behavior. + artifact := &v1.Artifact{ + NamespaceID: pid, + Repository: repository, + Digest: digest, + MimeType: v1.MimeTypeDockerArtifact, } - data, err := dao.GetImgScanOverview(digest) + + sum, err := scan.DefaultController.GetSummary(artifact, []string{v1.MimeTypeNativeReport}) if err != nil { - log.Errorf("Failed to get scan result for tag:%s, digest: %s, error: %v", tag, digest, err) + logger.Errorf("Failed to get scan report summary with error: %s", err) } - if data == nil { - return nil - } - job, err := dao.GetScanJob(data.JobID) - if err != nil { - log.Errorf("Failed to get scan job for id:%d, error: %v", data.JobID, err) - return nil - } else if job == nil { // job does not exist - log.Errorf("The scan job with id: %d does not exist, returning nil", data.JobID) - return nil - } - data.Status = job.Status - if data.Status != models.JobFinished { - log.Debugf("Unsetting vulnerable related historical values, job status: %s", data.Status) - data.Sev = 0 - data.CompOverview = nil - data.DetailsKey = "" - } - return data + + return sum } diff --git a/src/core/api/repository_test.go b/src/core/api/repository_test.go index 7aa17a0b2d..b51a38aeb2 100644 --- a/src/core/api/repository_test.go +++ b/src/core/api/repository_test.go @@ -42,7 +42,7 @@ func TestGetRepos(t *testing.T) { } else { assert.Equal(int(200), code, "response code should be 200") if repos, ok := repositories.([]repoResp); ok { - assert.Equal(int(1), len(repos), "the length of repositories should be 1") + require.Equal(t, int(1), len(repos), "the length of repositories should be 1") assert.Equal(repos[0].Name, "library/hello-world", "unexpected repository name") } else { t.Error("unexpected response") diff --git a/src/core/api/scan.go b/src/core/api/scan.go new file mode 100644 index 0000000000..aac29c22f4 --- /dev/null +++ b/src/core/api/scan.go @@ -0,0 +1,192 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "net/http" + "strconv" + + "github.com/goharbor/harbor/src/pkg/scan/report" + + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/rbac" + "github.com/goharbor/harbor/src/common/utils" + coreutils "github.com/goharbor/harbor/src/core/utils" + "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/pkg/scan/api/scan" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/pkg/errors" +) + +var digestFunc digestGetter = getDigest + +// ScanAPI handles the scan related actions +type ScanAPI struct { + BaseController + + // Target artifact + artifact *v1.Artifact + // Project reference + pro *models.Project +} + +// Prepare sth. for the subsequent actions +func (sa *ScanAPI) Prepare() { + // Call super prepare method + sa.BaseController.Prepare() + + // Parse parameters + repoName := sa.GetString(":splat") + tag := sa.GetString(":tag") + projectName, _ := utils.ParseRepository(repoName) + + pro, err := sa.ProjectMgr.Get(projectName) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scan API: prepare")) + return + } + if pro == nil { + sa.SendNotFoundError(errors.Errorf("project %s not found", projectName)) + return + } + sa.pro = pro + + // Check authentication + if !sa.RequireAuthenticated() { + return + } + + // Assemble artifact object + digest, err := digestFunc(repoName, tag, sa.SecurityCtx.GetUsername()) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scan API: prepare")) + return + } + + sa.artifact = &v1.Artifact{ + NamespaceID: pro.ProjectID, + Repository: repoName, + Tag: tag, + Digest: digest, + MimeType: v1.MimeTypeDockerArtifact, + } + + logger.Debugf("Scan API receives artifact: %#v", sa.artifact) +} + +// Scan artifact +func (sa *ScanAPI) Scan() { + // Check access permissions + if !sa.RequireProjectAccess(sa.pro.ProjectID, rbac.ActionCreate, rbac.ResourceScan) { + return + } + + if err := scan.DefaultController.Scan(sa.artifact); err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scan API: scan")) + return + } + + sa.Ctx.ResponseWriter.WriteHeader(http.StatusAccepted) +} + +// Report returns the required reports with the given mime types. +func (sa *ScanAPI) Report() { + // Check access permissions + if !sa.RequireProjectAccess(sa.pro.ProjectID, rbac.ActionRead, rbac.ResourceScan) { + return + } + + // Extract mime types + producesMimes := make([]string, 0) + if hl, ok := sa.Ctx.Request.Header[v1.HTTPAcceptHeader]; ok && len(hl) > 0 { + producesMimes = append(producesMimes, hl...) + } + + // Get the reports + reports, err := scan.DefaultController.GetReport(sa.artifact, producesMimes) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scan API: get report")) + return + } + + vulItems := make(map[string]interface{}) + for _, rp := range reports { + // Resolve scan report data only when it is ready + if len(rp.Report) == 0 { + continue + } + + vrp, err := report.ResolveData(rp.MimeType, []byte(rp.Report)) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scan API: get report")) + return + } + + vulItems[rp.MimeType] = vrp + } + + sa.Data["json"] = vulItems + sa.ServeJSON() +} + +// Log returns the log stream +func (sa *ScanAPI) Log() { + // Check access permissions + if !sa.RequireProjectAccess(sa.pro.ProjectID, rbac.ActionRead, rbac.ResourceScan) { + return + } + + uuid := sa.GetString(":uuid") + bytes, err := scan.DefaultController.GetScanLog(uuid) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scan API: log")) + return + } + + if bytes == nil { + // Not found + sa.SendNotFoundError(errors.Errorf("report with uuid %s does not exist", uuid)) + return + } + + sa.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(bytes))) + sa.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain") + _, err = sa.Ctx.ResponseWriter.Write(bytes) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scan API: log")) + } +} + +// digestGetter is a function template for getting digest. +// TODO: This can be removed if the registry access interface is ready. +type digestGetter func(repo, tag string, username string) (string, error) + +func getDigest(repo, tag string, username string) (string, error) { + client, err := coreutils.NewRepositoryClientForUI(username, repo) + if err != nil { + return "", err + } + + digest, exists, err := client.ManifestExist(tag) + if err != nil { + return "", err + } + + if !exists { + return "", errors.Errorf("tag %s does exist", tag) + } + + return digest, nil +} diff --git a/src/core/api/scan_job.go b/src/core/api/scan_job.go deleted file mode 100644 index 7cc38d61e8..0000000000 --- a/src/core/api/scan_job.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2018 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/rbac" - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/core/utils" - - "errors" - "fmt" - "net/http" - "strconv" - "strings" -) - -// ScanJobAPI handles request to /api/scanJobs/:id/log -type ScanJobAPI struct { - BaseController - jobID int64 - projectName string - jobUUID string -} - -// Prepare validates that whether user has read permission to the project of the repo the scan job scanned. -func (sj *ScanJobAPI) Prepare() { - sj.BaseController.Prepare() - if !sj.SecurityCtx.IsAuthenticated() { - sj.SendUnAuthorizedError(errors.New("UnAuthorized")) - return - } - id, err := sj.GetInt64FromPath(":id") - if err != nil { - sj.SendBadRequestError(errors.New("invalid ID")) - return - } - sj.jobID = id - - data, err := dao.GetScanJob(id) - if err != nil { - log.Errorf("Failed to load job data for job: %d, error: %v", id, err) - sj.SendInternalServerError(errors.New("Failed to get Job data")) - return - } - - projectName := strings.SplitN(data.Repository, "/", 2)[0] - if !sj.RequireProjectAccess(projectName, rbac.ActionRead, rbac.ResourceRepositoryTagScanJob) { - log.Errorf("User does not have read permission for project: %s", projectName) - return - } - sj.projectName = projectName - sj.jobUUID = data.UUID -} - -// GetLog ... -func (sj *ScanJobAPI) GetLog() { - logBytes, err := utils.GetJobServiceClient().GetJobLog(sj.jobUUID) - if err != nil { - sj.ParseAndHandleError(fmt.Sprintf("Failed to get job logs, uuid: %s, error: %v", sj.jobUUID, err), err) - return - } - sj.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(logBytes))) - sj.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain") - _, err = sj.Ctx.ResponseWriter.Write(logBytes) - if err != nil { - sj.SendInternalServerError(fmt.Errorf("Failed to write job logs, uuid: %s, error: %v", sj.jobUUID, err)) - } - -} diff --git a/src/core/api/scan_test.go b/src/core/api/scan_test.go new file mode 100644 index 0000000000..2e80a35d0f --- /dev/null +++ b/src/core/api/scan_test.go @@ -0,0 +1,214 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/stretchr/testify/require" + + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/pkg/scan/api/scan" + dscan "github.com/goharbor/harbor/src/pkg/scan/dao/scan" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +var scanBaseURL = "/api/repositories/library/hello-world/tags/latest/scan" + +// ScanAPITestSuite is the test suite for scan API. +type ScanAPITestSuite struct { + suite.Suite + + originalC scan.Controller + c *MockScanAPIController + + originalDigestGetter digestGetter + + artifact *v1.Artifact +} + +// TestScanAPI is the entry point of ScanAPITestSuite. +func TestScanAPI(t *testing.T) { + suite.Run(t, new(ScanAPITestSuite)) +} + +// SetupSuite prepares test env for suite. +func (suite *ScanAPITestSuite) SetupSuite() { + suite.artifact = &v1.Artifact{ + NamespaceID: (int64)(1), + Repository: "library/hello-world", + Tag: "latest", + Digest: "digest-code-001", + MimeType: v1.MimeTypeDockerArtifact, + } +} + +// SetupTest prepares test env for test cases. +func (suite *ScanAPITestSuite) SetupTest() { + suite.originalC = scan.DefaultController + suite.c = &MockScanAPIController{} + + scan.DefaultController = suite.c + + suite.originalDigestGetter = digestFunc + digestFunc = func(repo, tag string, username string) (s string, e error) { + return "digest-code-001", nil + } +} + +// TearDownTest ... +func (suite *ScanAPITestSuite) TearDownTest() { + scan.DefaultController = suite.originalC + digestFunc = suite.originalDigestGetter +} + +// TestScanAPIBase ... +func (suite *ScanAPITestSuite) TestScanAPIBase() { + suite.c.On("Scan", &v1.Artifact{}).Return(nil) + // Including general cases + cases := []*codeCheckingCase{ + // 401 + { + request: &testingRequest{ + url: scanBaseURL, + method: http.MethodGet, + }, + code: http.StatusUnauthorized, + }, + // 403 + { + request: &testingRequest{ + url: scanBaseURL, + method: http.MethodPost, + credential: projGuest, + }, + code: http.StatusForbidden, + }, + } + + runCodeCheckingCases(suite.T(), cases...) +} + +// TestScanAPIScan ... +func (suite *ScanAPITestSuite) TestScanAPIScan() { + suite.c.On("Scan", suite.artifact).Return(nil) + + // Including general cases + cases := []*codeCheckingCase{ + // 202 + { + request: &testingRequest{ + url: scanBaseURL, + method: http.MethodPost, + credential: projDeveloper, + }, + code: http.StatusAccepted, + }, + } + + runCodeCheckingCases(suite.T(), cases...) +} + +// TestScanAPIReport ... +func (suite *ScanAPITestSuite) TestScanAPIReport() { + suite.c.On("GetReport", suite.artifact, []string{v1.MimeTypeNativeReport}).Return([]*dscan.Report{}, nil) + + vulItems := make(map[string]interface{}) + + header := make(http.Header) + header.Add("Accept", v1.MimeTypeNativeReport) + err := handleAndParse( + &testingRequest{ + url: scanBaseURL, + method: http.MethodGet, + credential: projDeveloper, + header: header, + }, &vulItems) + require.NoError(suite.T(), err) +} + +// TestScanAPILog ... +func (suite *ScanAPITestSuite) TestScanAPILog() { + suite.c.On("GetScanLog", "the-uuid-001").Return([]byte(`{"log": "this is my log"}`), nil) + + logs := make(map[string]string) + err := handleAndParse( + &testingRequest{ + url: fmt.Sprintf("%s/%s", scanBaseURL, "the-uuid-001/log"), + method: http.MethodGet, + credential: projDeveloper, + }, &logs) + require.NoError(suite.T(), err) + assert.Condition(suite.T(), func() (success bool) { + success = len(logs) > 0 + return + }) +} + +// Mock things + +// MockScanAPIController ... +type MockScanAPIController struct { + mock.Mock +} + +// Scan ... +func (msc *MockScanAPIController) Scan(artifact *v1.Artifact) error { + args := msc.Called(artifact) + + return args.Error(0) +} + +func (msc *MockScanAPIController) GetReport(artifact *v1.Artifact, mimeTypes []string) ([]*dscan.Report, error) { + args := msc.Called(artifact, mimeTypes) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]*dscan.Report), args.Error(1) +} + +func (msc *MockScanAPIController) GetSummary(artifact *v1.Artifact, mimeTypes []string) (map[string]interface{}, error) { + args := msc.Called(artifact, mimeTypes) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(map[string]interface{}), args.Error(1) +} + +func (msc *MockScanAPIController) GetScanLog(uuid string) ([]byte, error) { + args := msc.Called(uuid) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]byte), args.Error(1) +} + +func (msc *MockScanAPIController) HandleJobHooks(trackID string, change *job.StatusChange) error { + args := msc.Called(trackID, change) + + return args.Error(0) +} diff --git a/src/core/api/scanners.go b/src/core/api/scanners.go index ee00888f8a..53321f64a2 100644 --- a/src/core/api/scanners.go +++ b/src/core/api/scanners.go @@ -62,6 +62,21 @@ func (sa *ScannerAPI) Get() { } } +// Metadata returns the metadata of the given scanner. +func (sa *ScannerAPI) Metadata() { + uuid := sa.GetStringFromPath(":uuid") + + meta, err := sa.c.GetMetadata(uuid) + if err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: get metadata")) + return + } + + // Response to the client + sa.Data["json"] = meta + sa.ServeJSON() +} + // List all the scanners func (sa *ScannerAPI) List() { p, pz, err := sa.GetPaginationParams() @@ -77,7 +92,7 @@ func (sa *ScannerAPI) List() { // Get query key words kws := make(map[string]interface{}) - properties := []string{"name", "description", "url"} + properties := []string{"name", "description", "url", "ex_name", "ex_url"} for _, k := range properties { kw := sa.GetString(k) if len(kw) > 0 { @@ -193,10 +208,6 @@ func (sa *ScannerAPI) Update() { // Delete the scanner func (sa *ScannerAPI) Delete() { uid := sa.GetStringFromPath(":uuid") - if len(uid) == 0 { - sa.SendBadRequestError(errors.New("missing uid")) - return - } deleted, err := sa.c.DeleteRegistration(uid) if err != nil { @@ -217,10 +228,6 @@ func (sa *ScannerAPI) Delete() { // SetAsDefault sets the given registration as default one func (sa *ScannerAPI) SetAsDefault() { uid := sa.GetStringFromPath(":uuid") - if len(uid) == 0 { - sa.SendBadRequestError(errors.New("missing uid")) - return - } m := make(map[string]interface{}) if err := sa.DecodeJSONReq(&m); err != nil { @@ -242,51 +249,22 @@ func (sa *ScannerAPI) SetAsDefault() { sa.SendForbiddenError(errors.Errorf("not supported: %#v", m)) } -// GetProjectScanner gets the project level scanner -func (sa *ScannerAPI) GetProjectScanner() { - pid, err := sa.GetInt64FromPath(":pid") - if err != nil { - sa.SendBadRequestError(errors.Wrap(err, "scanner API: get project scanners")) +// Ping the registration. +func (sa *ScannerAPI) Ping() { + r := &scanner.Registration{} + + if err := sa.DecodeJSONReq(r); err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: ping")) return } - r, err := sa.c.GetRegistrationByProject(pid) - if err != nil { - sa.SendInternalServerError(errors.Wrap(err, "scanner API: get project scanners")) + if err := r.Validate(false); err != nil { + sa.SendBadRequestError(errors.Wrap(err, "scanner API: ping")) return } - if r != nil { - sa.Data["json"] = r - } else { - sa.Data["json"] = make(map[string]interface{}) - } - - sa.ServeJSON() -} - -// SetProjectScanner sets the project level scanner -func (sa *ScannerAPI) SetProjectScanner() { - pid, err := sa.GetInt64FromPath(":pid") - if err != nil { - sa.SendBadRequestError(errors.Wrap(err, "scanner API: set project scanners")) - return - } - - body := make(map[string]string) - if err := sa.DecodeJSONReq(&body); err != nil { - sa.SendBadRequestError(errors.Wrap(err, "scanner API: set project scanners")) - return - } - - uuid, ok := body["uuid"] - if !ok || len(uuid) == 0 { - sa.SendBadRequestError(errors.New("missing scanner uuid when setting project scanner")) - return - } - - if err := sa.c.SetRegistrationByProject(pid, uuid); err != nil { - sa.SendInternalServerError(errors.Wrap(err, "scanner API: set project scanners")) + if _, err := sa.c.Ping(r); err != nil { + sa.SendInternalServerError(errors.Wrap(err, "scanner API: ping")) return } } @@ -294,10 +272,6 @@ func (sa *ScannerAPI) SetProjectScanner() { // get the specified scanner func (sa *ScannerAPI) get() *scanner.Registration { uid := sa.GetStringFromPath(":uuid") - if len(uid) == 0 { - sa.SendBadRequestError(errors.New("missing uid")) - return nil - } r, err := sa.c.GetRegistration(uid) if err != nil { diff --git a/src/core/api/scanners_test.go b/src/core/api/scanners_test.go index 02744788fb..598ae74590 100644 --- a/src/core/api/scanners_test.go +++ b/src/core/api/scanners_test.go @@ -19,6 +19,8 @@ import ( "net/http" "testing" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/goharbor/harbor/src/pkg/q" sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" @@ -256,45 +258,6 @@ func (suite *ScannerAPITestSuite) TestScannerAPISetDefault() { }) } -// TestScannerAPIProjectScanner tests the API of getting/setting project level scanner -func (suite *ScannerAPITestSuite) TestScannerAPIProjectScanner() { - suite.mockC.On("SetRegistrationByProject", int64(1), "uuid").Return(nil) - - // Set - body := make(map[string]interface{}, 1) - body["uuid"] = "uuid" - runCodeCheckingCases(suite.T(), &codeCheckingCase{ - request: &testingRequest{ - url: fmt.Sprintf("/api/projects/%d/scanner", 1), - method: http.MethodPut, - credential: sysAdmin, - bodyJSON: body, - }, - code: http.StatusOK, - }) - - r := &scanner.Registration{ - ID: 1004, - UUID: "uuid", - Name: "TestScannerAPIProjectScanner", - Description: "JUST FOR TEST", - URL: "https://a.b.c", - } - suite.mockC.On("GetRegistrationByProject", int64(1)).Return(r, nil) - - // Get - rr := &scanner.Registration{} - err := handleAndParse(&testingRequest{ - url: fmt.Sprintf("/api/projects/%d/scanner", 1), - method: http.MethodGet, - credential: sysAdmin, - }, rr) - require.NoError(suite.T(), err) - - assert.Equal(suite.T(), r.Name, rr.Name) - assert.Equal(suite.T(), r.UUID, rr.UUID) -} - func (suite *ScannerAPITestSuite) mockQuery(r *scanner.Registration) { kw := make(map[string]interface{}, 1) kw["name"] = r.Name @@ -385,3 +348,25 @@ func (m *MockScannerAPIController) GetRegistrationByProject(projectID int64) (*s return s.(*scanner.Registration), args.Error(1) } + +// Ping ... +func (m *MockScannerAPIController) Ping(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) { + args := m.Called(registration) + sam := args.Get(0) + if sam == nil { + return nil, args.Error(1) + } + + return sam.(*v1.ScannerAdapterMetadata), nil +} + +// GetMetadata ... +func (m *MockScannerAPIController) GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error) { + args := m.Called(registrationUUID) + sam := args.Get(0) + if sam == nil { + return nil, args.Error(1) + } + + return sam.(*v1.ScannerAdapterMetadata), nil +} diff --git a/src/core/api/user.go b/src/core/api/user.go index 952c6fc34c..3372bdee8f 100644 --- a/src/core/api/user.go +++ b/src/core/api/user.go @@ -52,7 +52,7 @@ type userSearch struct { Username string `json:"username"` } -type secretResp struct { +type secretReq struct { Secret string `json:"secret"` } @@ -405,8 +405,8 @@ func (ua *UserAPI) ChangePassword() { return } - if len(req.NewPassword) == 0 { - ua.SendBadRequestError(errors.New("empty new_password")) + if err := validateSecret(req.NewPassword); err != nil { + ua.SendBadRequestError(err) return } @@ -512,8 +512,8 @@ func (ua *UserAPI) ListUserPermissions() { return } -// GenCLISecret generates a new CLI secret and replace the old one -func (ua *UserAPI) GenCLISecret() { +// SetCLISecret handles request PUT /api/users/:id/cli_secret to update the CLI secret of the user +func (ua *UserAPI) SetCLISecret() { if ua.AuthMode != common.OIDCAuth { ua.SendPreconditionFailedError(errors.New("the auth mode has to be oidc auth")) return @@ -534,8 +534,17 @@ func (ua *UserAPI) GenCLISecret() { return } - sec := utils.GenerateRandomString() - encSec, err := utils.ReversibleEncrypt(sec, ua.secretKey) + s := &secretReq{} + if err := ua.DecodeJSONReq(s); err != nil { + ua.SendBadRequestError(err) + return + } + if err := validateSecret(s.Secret); err != nil { + ua.SendBadRequestError(err) + return + } + + encSec, err := utils.ReversibleEncrypt(s.Secret, ua.secretKey) if err != nil { log.Errorf("Failed to encrypt secret, error: %v", err) ua.SendInternalServerError(errors.New("failed to encrypt secret")) @@ -548,8 +557,6 @@ func (ua *UserAPI) GenCLISecret() { ua.SendInternalServerError(errors.New("failed to update secret in DB")) return } - ua.Data["json"] = secretResp{sec} - ua.ServeJSON() } func (ua *UserAPI) getOIDCUserInfo() (*models.OIDCUser, error) { @@ -588,12 +595,24 @@ func validate(user models.User) error { if utils.IsContainIllegalChar(user.Username, []string{",", "~", "#", "$", "%"}) { return fmt.Errorf("username contains illegal characters") } - if utils.IsIllegalLength(user.Password, 8, 20) { - return fmt.Errorf("password with illegal length") + + if err := validateSecret(user.Password); err != nil { + return err } + return commonValidate(user) } +func validateSecret(in string) error { + hasLower := regexp.MustCompile(`[a-z]`) + hasUpper := regexp.MustCompile(`[A-Z]`) + hasNumber := regexp.MustCompile(`[0-9]`) + if len(in) >= 8 && hasLower.MatchString(in) && hasUpper.MatchString(in) && hasNumber.MatchString(in) { + return nil + } + return errors.New("the password or secret must longer than 8 chars with at least 1 uppercase letter, 1 lowercase letter and 1 number") +} + // commonValidate validates email, realname, comment information when user register or change their profile func commonValidate(user models.User) error { diff --git a/src/core/api/user_test.go b/src/core/api/user_test.go index 88f35dd0d8..530af05000 100644 --- a/src/core/api/user_test.go +++ b/src/core/api/user_test.go @@ -380,8 +380,8 @@ func buildChangeUserPasswordURL(id int) string { func TestUsersUpdatePassword(t *testing.T) { fmt.Println("Testing Update User Password") - oldPassword := "old_password" - newPassword := "new_password" + oldPassword := "old_Passw0rd" + newPassword := "new_Passw0rd" user01 := models.User{ Username: "user01_for_testing_change_password", @@ -515,7 +515,7 @@ func TestUsersUpdatePassword(t *testing.T) { method: http.MethodPut, url: buildChangeUserPasswordURL(user01.UserID), bodyJSON: &passwordReq{ - NewPassword: "another_new_password", + NewPassword: "another_new_Passw0rd", }, credential: admin, }, @@ -642,3 +642,13 @@ func TestUsersCurrentPermissions(t *testing.T) { assert.Nil(err) assert.Equal(int(403), httpStatusCode, "httpStatusCode should be 403") } + +func TestValidateSecret(t *testing.T) { + assert.NotNil(t, validateSecret("")) + assert.NotNil(t, validateSecret("12345678")) + assert.NotNil(t, validateSecret("passw0rd")) + assert.NotNil(t, validateSecret("PASSW0RD")) + assert.NotNil(t, validateSecret("Sh0rt")) + assert.Nil(t, validateSecret("Passw0rd")) + assert.Nil(t, validateSecret("Thisis1Valid_password")) +} diff --git a/src/core/middlewares/util/util.go b/src/core/middlewares/util/util.go index 8a96242919..85a32ab4bb 100644 --- a/src/core/middlewares/util/util.go +++ b/src/core/middlewares/util/util.go @@ -40,7 +40,7 @@ import ( "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/promgr" "github.com/goharbor/harbor/src/pkg/scan/whitelist" - "github.com/opencontainers/go-digest" + digest "github.com/opencontainers/go-digest" ) type contextKey string @@ -346,6 +346,10 @@ func (pc PmsPolicyChecker) ContentTrustEnabled(name string) bool { log.Errorf("Unexpected error when getting the project, error: %v", err) return true } + if project == nil { + log.Debugf("project %s not found", name) + return false + } return project.ContentTrustEnabled() } diff --git a/src/core/middlewares/util/util_test.go b/src/core/middlewares/util/util_test.go index 2e6c9d6096..db2f8960a7 100644 --- a/src/core/middlewares/util/util_test.go +++ b/src/core/middlewares/util/util_test.go @@ -32,7 +32,7 @@ import ( notarytest "github.com/goharbor/harbor/src/common/utils/notary/test" testutils "github.com/goharbor/harbor/src/common/utils/test" "github.com/goharbor/harbor/src/core/config" - "github.com/opencontainers/go-digest" + digest "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -187,6 +187,9 @@ func TestPMSPolicyChecker(t *testing.T) { assert.True(t, projectVulnerableEnabled) assert.Equal(t, projectVulnerableSeverity, models.SevLow) assert.Empty(t, wl.Items) + + contentTrustFlag = GetPolicyChecker().ContentTrustEnabled("non_exist_project") + assert.False(t, contentTrustFlag) } func TestCopyResp(t *testing.T) { diff --git a/src/core/router.go b/src/core/router.go index b7ee2736da..6f38680b94 100755 --- a/src/core/router.go +++ b/src/core/router.go @@ -52,7 +52,7 @@ func initRouters() { beego.Router("/api/users/:id([0-9]+)/password", &api.UserAPI{}, "put:ChangePassword") beego.Router("/api/users/:id/permissions", &api.UserAPI{}, "get:ListUserPermissions") beego.Router("/api/users/:id/sysadmin", &api.UserAPI{}, "put:ToggleUserAdminRole") - beego.Router("/api/users/:id/gen_cli_secret", &api.UserAPI{}, "post:GenCLISecret") + beego.Router("/api/users/:id/cli_secret", &api.UserAPI{}, "put:SetCLISecret") beego.Router("/api/usergroups/?:ugid([0-9]+)", &api.UserGroupAPI{}) beego.Router("/api/ldap/ping", &api.LdapAPI{}, "post:Ping") beego.Router("/api/ldap/users/search", &api.LdapAPI{}, "get:Search") @@ -87,12 +87,9 @@ func initRouters() { beego.Router("/api/repositories/*/tags/:tag/labels", &api.RepositoryLabelAPI{}, "get:GetOfImage;post:AddToImage") beego.Router("/api/repositories/*/tags/:tag/labels/:id([0-9]+)", &api.RepositoryLabelAPI{}, "delete:RemoveFromImage") beego.Router("/api/repositories/*/tags", &api.RepositoryAPI{}, "get:GetTags;post:Retag") - beego.Router("/api/repositories/*/tags/:tag/scan", &api.RepositoryAPI{}, "post:ScanImage") - beego.Router("/api/repositories/*/tags/:tag/vulnerability/details", &api.RepositoryAPI{}, "Get:VulnerabilityDetails") beego.Router("/api/repositories/*/tags/:tag/manifest", &api.RepositoryAPI{}, "get:GetManifests") beego.Router("/api/repositories/*/signatures", &api.RepositoryAPI{}, "get:GetSignatures") beego.Router("/api/repositories/top", &api.RepositoryAPI{}, "get:GetTopRepos") - beego.Router("/api/jobs/scan/:id([0-9]+)/log", &api.ScanJobAPI{}, "get:GetLog") beego.Router("/api/system/gc", &api.GCAPI{}, "get:List") beego.Router("/api/system/gc/:id", &api.GCAPI{}, "get:GetGC") @@ -142,7 +139,6 @@ func initRouters() { // external service that hosted on harbor process: beego.Router("/service/notifications", ®istry.NotificationHandler{}) - beego.Router("/service/notifications/jobs/scan/:id([0-9]+)", &jobs.Handler{}, "post:HandleScan") beego.Router("/service/notifications/jobs/adminjob/:id([0-9]+)", &admin.Handler{}, "post:HandleAdminJob") beego.Router("/service/notifications/jobs/replication/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationScheduleJob") beego.Router("/service/notifications/jobs/replication/task/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationTask") @@ -201,8 +197,20 @@ func initRouters() { scannerAPI := &api.ScannerAPI{} beego.Router("/api/scanners", scannerAPI, "post:Create;get:List") beego.Router("/api/scanners/:uuid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault") + beego.Router("/api/scanners/:uuid/metadata", scannerAPI, "get:Metadata") + beego.Router("/api/scanners/ping", scannerAPI, "post:Ping") + // Add routes for project level scanner - beego.Router("/api/projects/:pid([0-9]+)/scanner", scannerAPI, "get:GetProjectScanner;put:SetProjectScanner") + proScannerAPI := &api.ProjectScannerAPI{} + beego.Router("/api/projects/:pid([0-9]+)/scanner", proScannerAPI, "get:GetProjectScanner;put:SetProjectScanner") + + // Add routes for scan + scanAPI := &api.ScanAPI{} + beego.Router("/api/repositories/*/tags/:tag/scan", scanAPI, "post:Scan;get:Report") + beego.Router("/api/repositories/*/tags/:tag/scan/:uuid/log", scanAPI, "get:Log") + + // Handle scan hook + beego.Router("/service/notifications/jobs/scan/:uuid", &jobs.Handler{}, "post:HandleScan") // Error pages beego.ErrorController(&controllers.ErrorController{}) diff --git a/src/core/service/notifications/jobs/handler.go b/src/core/service/notifications/jobs/handler.go index b383400b4c..a1599ddc1a 100755 --- a/src/core/service/notifications/jobs/handler.go +++ b/src/core/service/notifications/jobs/handler.go @@ -18,7 +18,8 @@ import ( "encoding/json" "time" - "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/pkg/scan/api/scan" + "github.com/goharbor/harbor/src/common/job" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/log" @@ -49,29 +50,36 @@ type Handler struct { rawStatus string checkIn string revision int64 + trackID string + change *jjob.StatusChange } // Prepare ... func (h *Handler) Prepare() { - id, err := h.GetInt64FromPath(":id") - if err != nil { - log.Errorf("Failed to get job ID, error: %v", err) - // Avoid job service from resending... - h.Abort("200") - return + h.trackID = h.GetStringFromPath(":uuid") + if len(h.trackID) == 0 { + id, err := h.GetInt64FromPath(":id") + if err != nil { + log.Errorf("Failed to get job ID, error: %v", err) + // Avoid job service from resending... + h.Abort("200") + return + } + h.id = id } - h.id = id + var data jjob.StatusChange - err = json.Unmarshal(h.Ctx.Input.CopyBody(1<<32), &data) + err := json.Unmarshal(h.Ctx.Input.CopyBody(1<<32), &data) if err != nil { - log.Errorf("Failed to decode job status change, job ID: %d, error: %v", id, err) + log.Errorf("Failed to decode job status change with error: %v", err) h.Abort("200") return } + h.change = &data h.rawStatus = data.Status status, ok := statusMap[data.Status] if !ok { - log.Debugf("drop the job status update event: job id-%d, status-%s", id, status) + log.Debugf("drop the job status update event: job id-%d/track id-%s, status-%s", h.id, h.trackID, status) h.Abort("200") return } @@ -84,7 +92,8 @@ func (h *Handler) Prepare() { // HandleScan handles the webhook of scan job func (h *Handler) HandleScan() { - log.Debugf("received san job status update event: job-%d, status-%s", h.id, h.status) + log.Debugf("received san job status update event: job-%d, status-%s, track_id-%s", h.id, h.status, h.trackID) + // Trigger image scan webhook event only for JobFinished and JobError status if h.status == models.JobFinished || h.status == models.JobError { e := &event.Event{} @@ -101,7 +110,7 @@ func (h *Handler) HandleScan() { } } - if err := dao.UpdateScanJobStatus(h.id, h.status); err != nil { + if err := scan.DefaultController.HandleJobHooks(h.trackID, h.change); err != nil { log.Errorf("Failed to update job status, id: %d, status: %s", h.id, h.status) h.SendInternalServerError(err) return diff --git a/src/jobservice/job/impl/replication/replication.go b/src/jobservice/job/impl/replication/replication.go index d7a35cdbf8..00e5bd72b3 100644 --- a/src/jobservice/job/impl/replication/replication.go +++ b/src/jobservice/job/impl/replication/replication.go @@ -44,6 +44,8 @@ import ( _ "github.com/goharbor/harbor/src/replication/adapter/aliacr" // register the Jfrog Artifactory adapter _ "github.com/goharbor/harbor/src/replication/adapter/jfrog" + // register the Quay.io adapter + _ "github.com/goharbor/harbor/src/replication/adapter/quayio" // register the Helm Hub adapter _ "github.com/goharbor/harbor/src/replication/adapter/helmhub" ) diff --git a/src/jobservice/runtime/bootstrap.go b/src/jobservice/runtime/bootstrap.go index 9a2bdd41fb..78c657605f 100644 --- a/src/jobservice/runtime/bootstrap.go +++ b/src/jobservice/runtime/bootstrap.go @@ -23,9 +23,6 @@ import ( "syscall" "time" - "github.com/gomodule/redigo/redis" - "github.com/pkg/errors" - "github.com/goharbor/harbor/src/jobservice/api" "github.com/goharbor/harbor/src/jobservice/common/utils" "github.com/goharbor/harbor/src/jobservice/config" @@ -45,7 +42,10 @@ import ( "github.com/goharbor/harbor/src/jobservice/worker" "github.com/goharbor/harbor/src/jobservice/worker/cworker" "github.com/goharbor/harbor/src/pkg/retention" + sc "github.com/goharbor/harbor/src/pkg/scan" "github.com/goharbor/harbor/src/pkg/scheduler" + "github.com/gomodule/redigo/redis" + "github.com/pkg/errors" ) const ( @@ -242,7 +242,7 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool( // Only for debugging and testing purpose job.SampleJob: (*sample.Job)(nil), // Functional jobs - job.ImageScanJob: (*scan.ClairJob)(nil), + job.ImageScanJob: (*sc.Job)(nil), job.ImageScanAllJob: (*scan.All)(nil), job.ImageGC: (*gc.GarbageCollector)(nil), job.Replication: (*replication.Replication)(nil), diff --git a/src/pkg/scan/api/scan/base_controller.go b/src/pkg/scan/api/scan/base_controller.go index b3d6f85050..514eb9ba51 100644 --- a/src/pkg/scan/api/scan/base_controller.go +++ b/src/pkg/scan/api/scan/base_controller.go @@ -15,44 +15,397 @@ package scan import ( + "fmt" + "time" + + "github.com/goharbor/harbor/src/common" + cj "github.com/goharbor/harbor/src/common/job" + jm "github.com/goharbor/harbor/src/common/job/models" + "github.com/goharbor/harbor/src/common/rbac" + "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/pkg/robot" + "github.com/goharbor/harbor/src/pkg/robot/model" + sca "github.com/goharbor/harbor/src/pkg/scan" + sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner" "github.com/goharbor/harbor/src/pkg/scan/dao/scan" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + "github.com/goharbor/harbor/src/pkg/scan/report" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/google/uuid" + "github.com/pkg/errors" ) +// DefaultController is a default singleton scan API controller. +var DefaultController = NewController() + +const ( + configRegistryEndpoint = "registryEndpoint" + configCoreInternalAddr = "coreInternalAddr" +) + +// uuidGenerator is a func template which is for generating UUID. +type uuidGenerator func() (string, error) + +// configGetter is a func template which is used to wrap the config management +// utility methods. +type configGetter func(cfg string) (string, error) + // basicController is default implementation of api.Controller interface type basicController struct { - // Client for talking to scanner adapter - client v1.Client + // Manage the scan report records + manager report.Manager + // Scanner controller + sc sc.Controller + // Robot account controller + rc robot.Controller + // Job service client + jc cj.Client + // UUID generator + uuid uuidGenerator + // Configuration getter func + config configGetter } // NewController news a scan API controller func NewController() Controller { - return &basicController{} + return &basicController{ + // New report manager + manager: report.NewManager(), + // Refer to the default scanner controller + sc: sc.DefaultController, + // Refer to the default robot account controller + rc: robot.RobotCtr, + // Refer to the default job service client + jc: cj.GlobalClient, + // Generate UUID with uuid lib + uuid: func() (string, error) { + aUUID, err := uuid.NewUUID() + if err != nil { + return "", err + } + + return aUUID.String(), nil + }, + // Get the required configuration options + config: func(cfg string) (string, error) { + switch cfg { + case configRegistryEndpoint: + return config.ExtEndpoint() + case configCoreInternalAddr: + return config.InternalCoreURL(), nil + default: + return "", errors.Errorf("configuration option %s not defined", cfg) + } + }, + } } // Scan ... func (bc *basicController) Scan(artifact *v1.Artifact) error { + if artifact == nil { + return errors.New("nil artifact to scan") + } + + r, err := bc.sc.GetRegistrationByProject(artifact.NamespaceID) + if err != nil { + return errors.Wrap(err, "scan controller: scan") + } + + // Check the health of the registration by ping. + // The metadata of the scanner adapter is also returned. + meta, err := bc.sc.Ping(r) + if err != nil { + return errors.Wrap(err, "scan controller: scan") + } + + // Generate a UUID as track ID which groups the report records generated + // by the specified registration for the digest with given mime type. + trackID, err := bc.uuid() + if err != nil { + return errors.Wrap(err, "scan controller: scan") + } + + producesMimes := make([]string, 0) + matched := false + for _, ca := range meta.Capabilities { + for _, cm := range ca.ConsumesMimeTypes { + if cm == artifact.MimeType { + matched = true + break + } + } + + if matched { + for _, pm := range ca.ProducesMimeTypes { + // Create report placeholder first + reportPlaceholder := &scan.Report{ + Digest: artifact.Digest, + RegistrationUUID: r.UUID, + Status: job.PendingStatus.String(), + StatusCode: job.PendingStatus.Code(), + TrackID: trackID, + MimeType: pm, + } + _, e := bc.manager.Create(reportPlaceholder) + if e != nil { + // Recorded by error wrap and logged at the same time. + if err == nil { + err = e + } else { + err = errors.Wrap(e, err.Error()) + } + + logger.Error(errors.Wrap(e, "scan controller: scan")) + continue + } + + producesMimes = append(producesMimes, pm) + } + + break + } + } + + // Scanner does not support scanning the given artifact. + if !matched { + return errors.Errorf("the configured scanner %s does not support scanning artifact with mime type %s", r.Name, artifact.MimeType) + } + + // If all the record are created failed. + if len(producesMimes) == 0 { + // Return the last error + return errors.Wrap(err, "scan controller: scan") + } + + jobID, err := bc.launchScanJob(trackID, artifact, r, producesMimes) + if err != nil { + // Update the status to the concrete error + // Change status code to normal error code + if e := bc.manager.UpdateStatus(trackID, err.Error(), 0); e != nil { + err = errors.Wrap(e, err.Error()) + } + + return errors.Wrap(err, "scan controller: scan") + } + + // Insert the generated job ID now + // It will not block the whole process. If any errors happened, just logged. + if err := bc.manager.UpdateScanJobID(trackID, jobID); err != nil { + logger.Error(errors.Wrap(err, "scan controller: scan")) + } + return nil } // GetReport ... -func (bc *basicController) GetReport(artifact *v1.Artifact) ([]*scan.Report, error) { - return nil, nil +func (bc *basicController) GetReport(artifact *v1.Artifact, mimeTypes []string) ([]*scan.Report, error) { + if artifact == nil { + return nil, errors.New("no way to get report for nil artifact") + } + + mimes := make([]string, 0) + mimes = append(mimes, mimeTypes...) + if len(mimes) == 0 { + // Retrieve native as default + mimes = append(mimes, v1.MimeTypeNativeReport) + } + + // Get current scanner settings + r, err := bc.sc.GetRegistrationByProject(artifact.NamespaceID) + if err != nil { + return nil, errors.Wrap(err, "scan controller: get report") + } + + if r == nil { + return nil, errors.New("no scanner registration configured") + } + + return bc.manager.GetBy(artifact.Digest, r.UUID, mimes) +} + +// GetSummary ... +func (bc *basicController) GetSummary(artifact *v1.Artifact, mimeTypes []string) (map[string]interface{}, error) { + if artifact == nil { + return nil, errors.New("no way to get report summaries for nil artifact") + } + + // Get reports first + rps, err := bc.GetReport(artifact, mimeTypes) + if err != nil { + return nil, err + } + + summaries := make(map[string]interface{}, len(rps)) + for _, rp := range rps { + sum, err := report.GenerateSummary(rp) + if err != nil { + return nil, err + } + + summaries[rp.MimeType] = sum + } + + return summaries, nil } // GetScanLog ... -func (bc *basicController) GetScanLog(digest string) ([]byte, error) { - return nil, nil -} +func (bc *basicController) GetScanLog(uuid string) ([]byte, error) { + if len(uuid) == 0 { + return nil, errors.New("empty uuid to get scan log") + } -// Ping ... -func (bc *basicController) Ping(registration *scanner.Registration) error { - return nil + // Get by uuid + sr, err := bc.manager.Get(uuid) + if err != nil { + return nil, errors.Wrap(err, "scan controller: get scan log") + } + + if sr == nil { + // Not found + return nil, nil + } + + // Not job error + if sr.StatusCode == job.ErrorStatus.Code() { + jst := job.Status(sr.Status) + if jst.Code() == -1 { + return []byte(sr.Status), nil + } + } + + // Job log + return bc.jc.GetJobLog(sr.JobID) } // HandleJobHooks ... -func (bc *basicController) HandleJobHooks(trackID int64, change *job.StatusChange) error { - return nil +func (bc *basicController) HandleJobHooks(trackID string, change *job.StatusChange) error { + if len(trackID) == 0 { + return errors.New("empty track ID") + } + + if change == nil { + return errors.New("nil change object") + } + + // Check in data + if len(change.CheckIn) > 0 { + checkInReport := &sca.CheckInReport{} + if err := checkInReport.FromJSON(change.CheckIn); err != nil { + return errors.Wrap(err, "scan controller: handle job hook") + } + + rpl, err := bc.manager.GetBy( + checkInReport.Digest, + checkInReport.RegistrationUUID, + []string{checkInReport.MimeType}) + if err != nil { + return errors.Wrap(err, "scan controller: handle job hook") + } + + if len(rpl) == 0 { + return errors.New("no report found to update data") + } + + if err := bc.manager.UpdateReportData( + rpl[0].UUID, + checkInReport.RawReport, + change.Metadata.Revision); err != nil { + return errors.Wrap(err, "scan controller: handle job hook") + } + + return nil + } + + return bc.manager.UpdateStatus(trackID, change.Status, change.Metadata.Revision) +} + +// makeRobotAccount creates a robot account based on the arguments for scanning. +func (bc *basicController) makeRobotAccount(pid int64, repository string, ttl int64) (string, error) { + // Use uuid as name to avoid duplicated entries. + UUID, err := bc.uuid() + if err != nil { + return "", errors.Wrap(err, "scan controller: make robot account") + } + + expireAt := time.Now().UTC().Add(time.Duration(ttl) * time.Second).Unix() + + logger.Warningf("repository %s and expire time %d are not supported by robot controller", repository, expireAt) + + resource := fmt.Sprintf("/project/%d/repository", pid) + access := []*rbac.Policy{{ + Resource: rbac.Resource(resource), + Action: "pull", + }} + + account := &model.RobotCreate{ + Name: fmt.Sprintf("%s%s", common.RobotPrefix, UUID), + Description: "for scan", + ProjectID: pid, + Access: access, + } + + rb, err := bc.rc.CreateRobotAccount(account) + if err != nil { + return "", errors.Wrap(err, "scan controller: make robot account") + } + + return rb.Token, nil +} + +// launchScanJob launches a job to run scan +func (bc *basicController) launchScanJob(trackID string, artifact *v1.Artifact, registration *scanner.Registration, mimes []string) (jobID string, err error) { + externalURL, err := bc.config(configRegistryEndpoint) + if err != nil { + return "", errors.Wrap(err, "scan controller: launch scan job") + } + + // Make a robot account with 30 minutes + robotAccount, err := bc.makeRobotAccount(artifact.NamespaceID, artifact.Repository, 1800) + if err != nil { + return "", errors.Wrap(err, "scan controller: launch scan job") + } + + // Set job parameters + scanReq := &v1.ScanRequest{ + Registry: &v1.Registry{ + URL: externalURL, + Authorization: robotAccount, + }, + Artifact: artifact, + } + + rJSON, err := registration.ToJSON() + if err != nil { + return "", errors.Wrap(err, "scan controller: launch scan job") + } + + sJSON, err := scanReq.ToJSON() + if err != nil { + return "", errors.Wrap(err, "launch scan job") + } + + params := make(map[string]interface{}) + params[sca.JobParamRegistration] = rJSON + params[sca.JobParameterRequest] = sJSON + params[sca.JobParameterMimes] = mimes + + // Launch job + callbackURL, err := bc.config(configCoreInternalAddr) + if err != nil { + return "", errors.Wrap(err, "launch scan job") + } + hookURL := fmt.Sprintf("%s/service/notifications/jobs/scan/%s", callbackURL, trackID) + + j := &jm.JobData{ + Name: job.ImageScanJob, + Metadata: &jm.JobMetadata{ + JobKind: job.KindGeneric, + }, + Parameters: params, + StatusHook: hookURL, + } + + return bc.jc.SubmitJob(j) } diff --git a/src/pkg/scan/api/scan/base_controller_test.go b/src/pkg/scan/api/scan/base_controller_test.go new file mode 100644 index 0000000000..932793f106 --- /dev/null +++ b/src/pkg/scan/api/scan/base_controller_test.go @@ -0,0 +1,537 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/rbac" + + "github.com/goharbor/harbor/src/pkg/robot/model" + + cjm "github.com/goharbor/harbor/src/common/job/models" + jm "github.com/goharbor/harbor/src/common/job/models" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/pkg/q" + sca "github.com/goharbor/harbor/src/pkg/scan" + "github.com/goharbor/harbor/src/pkg/scan/dao/scan" + "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/goharbor/harbor/src/pkg/scan/vuln" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// ControllerTestSuite is the test suite for scan controller. +type ControllerTestSuite struct { + suite.Suite + + registration *scanner.Registration + artifact *v1.Artifact + rawReport string + c Controller +} + +// TestController is the entry point of ControllerTestSuite. +func TestController(t *testing.T) { + suite.Run(t, new(ControllerTestSuite)) +} + +// SetupSuite ... +func (suite *ControllerTestSuite) SetupSuite() { + suite.registration = &scanner.Registration{ + ID: 1, + UUID: "uuid001", + Name: "Test-scan-controller", + URL: "http://testing.com:3128", + IsDefault: true, + } + + suite.artifact = &v1.Artifact{ + NamespaceID: 1, + Repository: "scan", + Tag: "golang", + Digest: "digest-code", + MimeType: v1.MimeTypeDockerArtifact, + } + + m := &v1.ScannerAdapterMetadata{ + Scanner: &v1.Scanner{ + Name: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + }, + Capabilities: []*v1.ScannerCapability{{ + ConsumesMimeTypes: []string{ + v1.MimeTypeOCIArtifact, + v1.MimeTypeDockerArtifact, + }, + ProducesMimeTypes: []string{ + v1.MimeTypeNativeReport, + }, + }}, + Properties: v1.ScannerProperties{ + "extra": "testing", + }, + } + + sc := &MockScannerController{} + sc.On("GetRegistrationByProject", suite.artifact.NamespaceID).Return(suite.registration, nil) + sc.On("Ping", suite.registration).Return(m, nil) + + mgr := &MockReportManager{} + mgr.On("Create", &scan.Report{ + Digest: "digest-code", + RegistrationUUID: "uuid001", + MimeType: "application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0", + Status: "Pending", + StatusCode: 0, + TrackID: "the-uuid-123", + }).Return("r-uuid", nil) + mgr.On("UpdateScanJobID", "the-uuid-123", "the-job-id").Return(nil) + + rp := vuln.Report{ + GeneratedAt: time.Now().UTC().String(), + Scanner: &v1.Scanner{ + Name: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + }, + Severity: vuln.High, + Vulnerabilities: []*vuln.VulnerabilityItem{ + { + ID: "2019-0980-0909", + Package: "dpkg", + Version: "0.9.1", + FixVersion: "0.9.2", + Severity: vuln.High, + Description: "mock one", + Links: []string{"https://vuln.com"}, + }, + }, + } + + jsonData, err := json.Marshal(rp) + require.NoError(suite.T(), err) + suite.rawReport = string(jsonData) + + reports := []*scan.Report{ + { + ID: 11, + UUID: "rp-uuid-001", + Digest: "digest-code", + RegistrationUUID: "uuid001", + MimeType: "application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0", + Status: "Success", + StatusCode: 3, + TrackID: "the-uuid-123", + JobID: "the-job-id", + StatusRevision: time.Now().Unix(), + Report: suite.rawReport, + StartTime: time.Now(), + EndTime: time.Now().Add(2 * time.Second), + }, + } + + mgr.On("GetBy", suite.artifact.Digest, suite.registration.UUID, []string{v1.MimeTypeNativeReport}).Return(reports, nil) + mgr.On("Get", "rp-uuid-001").Return(reports[0], nil) + mgr.On("UpdateReportData", "rp-uuid-001", suite.rawReport, (int64)(10000)).Return(nil) + mgr.On("UpdateStatus", "the-uuid-123", "Success", (int64)(10000)).Return(nil) + + rc := &MockRobotController{} + + resource := fmt.Sprintf("/project/%d/repository", suite.artifact.NamespaceID) + access := []*rbac.Policy{{ + Resource: rbac.Resource(resource), + Action: "pull", + }} + + rname := fmt.Sprintf("%s%s", common.RobotPrefix, "the-uuid-123") + account := &model.RobotCreate{ + Name: rname, + Description: "for scan", + ProjectID: suite.artifact.NamespaceID, + Access: access, + } + rc.On("CreateRobotAccount", account).Return(&model.Robot{ + ID: 1, + Name: rname, + Token: "robot-account", + Description: "for scan", + ProjectID: suite.artifact.NamespaceID, + }, nil) + + // Set job parameters + req := &v1.ScanRequest{ + Registry: &v1.Registry{ + URL: "https://core.com", + Authorization: "robot-account", + }, + Artifact: suite.artifact, + } + + rJSON, err := req.ToJSON() + require.NoError(suite.T(), err) + + regJSON, err := suite.registration.ToJSON() + require.NoError(suite.T(), err) + + jc := &MockJobServiceClient{} + params := make(map[string]interface{}) + params[sca.JobParamRegistration] = regJSON + params[sca.JobParameterRequest] = rJSON + params[sca.JobParameterMimes] = []string{v1.MimeTypeNativeReport} + + j := &jm.JobData{ + Name: job.ImageScanJob, + Metadata: &jm.JobMetadata{ + JobKind: job.KindGeneric, + }, + Parameters: params, + StatusHook: fmt.Sprintf("%s/service/notifications/jobs/scan/%s", "http://core:8080", "the-uuid-123"), + } + jc.On("SubmitJob", j).Return("the-job-id", nil) + jc.On("GetJobLog", "the-job-id").Return([]byte("job log"), nil) + + suite.c = &basicController{ + manager: mgr, + sc: sc, + jc: jc, + rc: rc, + uuid: func() (string, error) { + return "the-uuid-123", nil + }, + config: func(cfg string) (string, error) { + switch cfg { + case configRegistryEndpoint: + return "https://core.com", nil + case configCoreInternalAddr: + return "http://core:8080", nil + } + + return "", nil + }, + } +} + +// TearDownSuite ... +func (suite *ControllerTestSuite) TearDownSuite() {} + +// TestScanControllerScan ... +func (suite *ControllerTestSuite) TestScanControllerScan() { + err := suite.c.Scan(suite.artifact) + require.NoError(suite.T(), err) +} + +// TestScanControllerGetReport ... +func (suite *ControllerTestSuite) TestScanControllerGetReport() { + rep, err := suite.c.GetReport(suite.artifact, []string{v1.MimeTypeNativeReport}) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), 1, len(rep)) +} + +// TestScanControllerGetSummary ... +func (suite *ControllerTestSuite) TestScanControllerGetSummary() { + sum, err := suite.c.GetSummary(suite.artifact, []string{v1.MimeTypeNativeReport}) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), 1, len(sum)) +} + +// TestScanControllerGetScanLog ... +func (suite *ControllerTestSuite) TestScanControllerGetScanLog() { + bytes, err := suite.c.GetScanLog("rp-uuid-001") + require.NoError(suite.T(), err) + assert.Condition(suite.T(), func() (success bool) { + success = len(bytes) > 0 + return + }) +} + +// TestScanControllerHandleJobHooks ... +func (suite *ControllerTestSuite) TestScanControllerHandleJobHooks() { + cReport := &sca.CheckInReport{ + Digest: "digest-code", + RegistrationUUID: suite.registration.UUID, + MimeType: v1.MimeTypeNativeReport, + RawReport: suite.rawReport, + } + + cRpJSON, err := cReport.ToJSON() + require.NoError(suite.T(), err) + + statusChange := &job.StatusChange{ + JobID: "the-job-id", + Status: "Success", + CheckIn: string(cRpJSON), + Metadata: &job.StatsInfo{ + Revision: (int64)(10000), + }, + } + + err = suite.c.HandleJobHooks("the-uuid-123", statusChange) + require.NoError(suite.T(), err) +} + +// Mock things + +// MockReportManager ... +type MockReportManager struct { + mock.Mock +} + +// Create ... +func (mrm *MockReportManager) Create(r *scan.Report) (string, error) { + args := mrm.Called(r) + + return args.String(0), args.Error(1) +} + +// UpdateScanJobID ... +func (mrm *MockReportManager) UpdateScanJobID(trackID string, jobID string) error { + args := mrm.Called(trackID, jobID) + + return args.Error(0) +} + +func (mrm *MockReportManager) UpdateStatus(trackID string, status string, rev int64) error { + args := mrm.Called(trackID, status, rev) + + return args.Error(0) +} + +func (mrm *MockReportManager) UpdateReportData(uuid string, report string, rev int64) error { + args := mrm.Called(uuid, report, rev) + + return args.Error(0) +} + +func (mrm *MockReportManager) GetBy(digest string, registrationUUID string, mimeTypes []string) ([]*scan.Report, error) { + args := mrm.Called(digest, registrationUUID, mimeTypes) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]*scan.Report), args.Error(1) +} + +func (mrm *MockReportManager) Get(uuid string) (*scan.Report, error) { + args := mrm.Called(uuid) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*scan.Report), args.Error(1) +} + +// MockScannerController ... +type MockScannerController struct { + mock.Mock +} + +// ListRegistrations ... +func (msc *MockScannerController) ListRegistrations(query *q.Query) ([]*scanner.Registration, error) { + args := msc.Called(query) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]*scanner.Registration), args.Error(1) +} + +// CreateRegistration ... +func (msc *MockScannerController) CreateRegistration(registration *scanner.Registration) (string, error) { + args := msc.Called(registration) + + return args.String(0), args.Error(1) +} + +// GetRegistration ... +func (msc *MockScannerController) GetRegistration(registrationUUID string) (*scanner.Registration, error) { + args := msc.Called(registrationUUID) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*scanner.Registration), args.Error(1) +} + +// RegistrationExists ... +func (msc *MockScannerController) RegistrationExists(registrationUUID string) bool { + args := msc.Called(registrationUUID) + + return args.Bool(0) +} + +// UpdateRegistration ... +func (msc *MockScannerController) UpdateRegistration(registration *scanner.Registration) error { + args := msc.Called(registration) + + return args.Error(0) +} + +// DeleteRegistration ... +func (msc *MockScannerController) DeleteRegistration(registrationUUID string) (*scanner.Registration, error) { + args := msc.Called(registrationUUID) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*scanner.Registration), args.Error(1) +} + +// SetDefaultRegistration ... +func (msc *MockScannerController) SetDefaultRegistration(registrationUUID string) error { + args := msc.Called(registrationUUID) + + return args.Error(0) +} + +// SetRegistrationByProject ... +func (msc *MockScannerController) SetRegistrationByProject(projectID int64, scannerID string) error { + args := msc.Called(projectID, scannerID) + + return args.Error(0) +} + +// GetRegistrationByProject ... +func (msc *MockScannerController) GetRegistrationByProject(projectID int64) (*scanner.Registration, error) { + args := msc.Called(projectID) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*scanner.Registration), args.Error(1) +} + +// Ping ... +func (msc *MockScannerController) Ping(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) { + args := msc.Called(registration) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*v1.ScannerAdapterMetadata), args.Error(1) +} + +// GetMetadata ... +func (msc *MockScannerController) GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error) { + args := msc.Called(registrationUUID) + + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*v1.ScannerAdapterMetadata), args.Error(1) +} + +// MockJobServiceClient ... +type MockJobServiceClient struct { + mock.Mock +} + +// SubmitJob ... +func (mjc *MockJobServiceClient) SubmitJob(jData *cjm.JobData) (string, error) { + args := mjc.Called(jData) + + return args.String(0), args.Error(1) +} + +// GetJobLog ... +func (mjc *MockJobServiceClient) GetJobLog(uuid string) ([]byte, error) { + args := mjc.Called(uuid) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]byte), args.Error(1) +} + +// PostAction ... +func (mjc *MockJobServiceClient) PostAction(uuid, action string) error { + args := mjc.Called(uuid, action) + + return args.Error(0) +} + +func (mjc *MockJobServiceClient) GetExecutions(uuid string) ([]job.Stats, error) { + args := mjc.Called(uuid) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]job.Stats), args.Error(1) +} + +// MockRobotController ... +type MockRobotController struct { + mock.Mock +} + +// GetRobotAccount ... +func (mrc *MockRobotController) GetRobotAccount(id int64) (*model.Robot, error) { + args := mrc.Called(id) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*model.Robot), args.Error(1) +} + +// CreateRobotAccount ... +func (mrc *MockRobotController) CreateRobotAccount(robotReq *model.RobotCreate) (*model.Robot, error) { + args := mrc.Called(robotReq) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*model.Robot), args.Error(1) +} + +// DeleteRobotAccount ... +func (mrc *MockRobotController) DeleteRobotAccount(id int64) error { + args := mrc.Called(id) + + return args.Error(0) +} + +// UpdateRobotAccount ... +func (mrc *MockRobotController) UpdateRobotAccount(r *model.Robot) error { + args := mrc.Called(r) + + return args.Error(0) +} + +// ListRobotAccount ... +func (mrc *MockRobotController) ListRobotAccount(pid int64) ([]*model.Robot, error) { + args := mrc.Called(pid) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]*model.Robot), args.Error(1) +} diff --git a/src/pkg/scan/api/scan/controller.go b/src/pkg/scan/api/scan/controller.go index 65322148be..69752c403e 100644 --- a/src/pkg/scan/api/scan/controller.go +++ b/src/pkg/scan/api/scan/controller.go @@ -17,7 +17,6 @@ package scan import ( "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/pkg/scan/dao/scan" - "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" ) @@ -25,17 +24,6 @@ import ( // TODO: Here the artifact object is reused the v1 one which is sent to the adapter, // it should be pointed to the general artifact object in future once it's ready. type Controller interface { - // Ping pings Scanner Adapter to test EndpointURL and Authorization settings. - // The implementation is supposed to call the GetMetadata method on scanner.Client. - // Returns `nil` if connection succeeded, a non `nil` error otherwise. - // - // Arguments: - // registration *scanner.Registration : scanner registration to ping - // - // Returns: - // error : non nil error if any errors occurred - Ping(registration *scanner.Registration) error - // Scan the given artifact // // Arguments: @@ -49,30 +37,42 @@ type Controller interface { // // Arguments: // artifact *v1.Artifact : the scanned artifact + // mimeTypes []string : the mime types of the reports // // Returns: // []*scan.Report : scan results by different scanner vendors // error : non nil error if any errors occurred - GetReport(artifact *v1.Artifact) ([]*scan.Report, error) + GetReport(artifact *v1.Artifact, mimeTypes []string) ([]*scan.Report, error) + + // GetSummary gets the summaries of the reports with given types. + // + // Arguments: + // artifact *v1.Artifact : the scanned artifact + // mimeTypes []string : the mime types of the reports + // + // Returns: + // map[string]interface{} : report summaries indexed by mime types + // error : non nil error if any errors occurred + GetSummary(artifact *v1.Artifact, mimeTypes []string) (map[string]interface{}, error) // Get the scan log for the specified artifact with the given digest // // Arguments: - // digest string : the digest of the artifact + // uuid string : the UUID of the scan report // // Returns: // []byte : the log text stream // error : non nil error if any errors occurred - GetScanLog(digest string) ([]byte, error) + GetScanLog(uuid string) ([]byte, error) // HandleJobHooks handle the hook events from the job service // e.g : status change of the scan job or scan result // // Arguments: - // trackID int64 : ID for the result record + // trackID string : UUID for the report record // change *job.StatusChange : change event from the job service // // Returns: // error : non nil error if any errors occurred - HandleJobHooks(trackID int64, change *job.StatusChange) error + HandleJobHooks(trackID string, change *job.StatusChange) error } diff --git a/src/pkg/scan/api/scanner/base_controller.go b/src/pkg/scan/api/scanner/base_controller.go index 72512ceff7..2eb4688c4e 100644 --- a/src/pkg/scan/api/scanner/base_controller.go +++ b/src/pkg/scan/api/scanner/base_controller.go @@ -19,6 +19,7 @@ import ( "github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/pkg/q" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" rscanner "github.com/goharbor/harbor/src/pkg/scan/scanner" "github.com/pkg/errors" ) @@ -35,25 +36,38 @@ func New() Controller { return &basicController{ manager: rscanner.New(), proMetaMgr: metamgr.NewDefaultProjectMetadataManager(), + clientPool: v1.DefaultClientPool, } } // basicController is default implementation of api.Controller interface type basicController struct { - // managers for managing the scanner registrations + // Managers for managing the scanner registrations manager rscanner.Manager - // for operating the project level configured scanner + // For operating the project level configured scanner proMetaMgr metamgr.ProjectMetadataManager + // Client pool for talking to adapters + clientPool v1.ClientPool } // ListRegistrations ... func (bc *basicController) ListRegistrations(query *q.Query) ([]*scanner.Registration, error) { - return bc.manager.List(query) + l, err := bc.manager.List(query) + if err != nil { + return nil, errors.Wrap(err, "api controller: list registrations") + } + + for _, r := range l { + _, err = bc.Ping(r) + r.Health = err == nil + } + + return l, nil } // CreateRegistration ... func (bc *basicController) CreateRegistration(registration *scanner.Registration) (string, error) { - // TODO: Get metadata from the adapter service first + // TODO: Check connection of the registration. // Check if there are any registrations already existing. l, err := bc.manager.List(&q.Query{ PageSize: 1, @@ -73,7 +87,15 @@ func (bc *basicController) CreateRegistration(registration *scanner.Registration // GetRegistration ... func (bc *basicController) GetRegistration(registrationUUID string) (*scanner.Registration, error) { - return bc.manager.Get(registrationUUID) + r, err := bc.manager.Get(registrationUUID) + if err != nil { + return nil, err + } + + _, err = bc.Ping(r) + r.Health = err == nil + + return r, nil } // RegistrationExists ... @@ -90,6 +112,10 @@ func (bc *basicController) RegistrationExists(registrationUUID string) bool { // UpdateRegistration ... func (bc *basicController) UpdateRegistration(registration *scanner.Registration) error { + if registration.IsDefault && registration.Disabled { + return errors.Errorf("default registration %s can not be marked to disabled", registration.UUID) + } + return bc.manager.Update(registration) } @@ -162,9 +188,10 @@ func (bc *basicController) GetRegistrationByProject(projectID int64) (*scanner.R return nil, errors.Wrap(err, "api controller: get project scanner") } + var registration *scanner.Registration if len(m) > 0 { if registrationID, ok := m[proScannerMetaKey]; ok && len(registrationID) > 0 { - registration, err := bc.manager.Get(registrationID) + registration, err = bc.manager.Get(registrationID) if err != nil { return nil, errors.Wrap(err, "api controller: get project scanner") } @@ -175,15 +202,103 @@ func (bc *basicController) GetRegistrationByProject(projectID int64) (*scanner.R if err := bc.proMetaMgr.Delete(projectID, proScannerMetaKey); err != nil { return nil, errors.Wrap(err, "api controller: get project scanner") } - } else { - return registration, nil } } } - // Second, get the default one - registration, err := bc.manager.GetDefault() + if registration == nil { + // Second, get the default one + registration, err = bc.manager.GetDefault() + if err != nil { + return nil, errors.Wrap(err, "api controller: get project scanner") + } + } + + // Check status by the client later + if registration != nil { + if meta, err := bc.Ping(registration); err == nil { + registration.Scanner = meta.Scanner.Name + registration.Vendor = meta.Scanner.Vendor + registration.Version = meta.Scanner.Version + registration.Health = true + } else { + registration.Health = false + } + } - // TODO: Check status by the client later return registration, err } + +// Ping ... +// TODO: ADD UT CASES +func (bc *basicController) Ping(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) { + if registration == nil { + return nil, errors.New("nil registration to ping") + } + + client, err := bc.clientPool.Get(registration) + if err != nil { + return nil, errors.Wrap(err, "scanner controller: ping") + } + + meta, err := client.GetMetadata() + if err != nil { + return nil, errors.Wrap(err, "scanner controller: ping") + } + + // Validate the required properties + if meta.Scanner == nil || + len(meta.Scanner.Name) == 0 || + len(meta.Scanner.Version) == 0 || + len(meta.Scanner.Vendor) == 0 { + return nil, errors.New("invalid scanner in metadata") + } + + if len(meta.Capabilities) == 0 { + return nil, errors.New("invalid capabilities in metadata") + } + + for _, ca := range meta.Capabilities { + // v1.MimeTypeDockerArtifact is required now + found := false + for _, cm := range ca.ConsumesMimeTypes { + if cm == v1.MimeTypeDockerArtifact { + found = true + break + } + } + if !found { + return nil, errors.Errorf("missing %s in consumes_mime_types", v1.MimeTypeDockerArtifact) + } + + // v1.MimeTypeNativeReport is required + found = false + for _, pm := range ca.ProducesMimeTypes { + if pm == v1.MimeTypeNativeReport { + found = true + break + } + } + + if !found { + return nil, errors.Errorf("missing %s in produces_mime_types", v1.MimeTypeNativeReport) + } + } + + return meta, err +} + +// GetMetadata ... +// TODO: ADD UT CASES +func (bc *basicController) GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error) { + if len(registrationUUID) == 0 { + return nil, errors.New("empty registration uuid") + } + + r, err := bc.manager.Get(registrationUUID) + if err != nil { + return nil, errors.Wrap(err, "scanner controller: get metadata") + } + + return bc.Ping(r) +} diff --git a/src/pkg/scan/api/scanner/controller_test.go b/src/pkg/scan/api/scanner/base_controller_test.go similarity index 82% rename from src/pkg/scan/api/scanner/controller_test.go rename to src/pkg/scan/api/scanner/base_controller_test.go index e032b645db..eef26baf74 100644 --- a/src/pkg/scan/api/scanner/controller_test.go +++ b/src/pkg/scan/api/scanner/base_controller_test.go @@ -17,6 +17,8 @@ package scanner import ( "testing" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/pkg/q" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" @@ -47,9 +49,25 @@ func (suite *ControllerTestSuite) SetupSuite() { suite.mMgr = new(MockScannerManager) suite.mMeta = new(MockProMetaManager) - suite.c = &basicController{ - manager: suite.mMgr, - proMetaMgr: suite.mMeta, + m := &v1.ScannerAdapterMetadata{ + Scanner: &v1.Scanner{ + Name: "Clair", + Vendor: "Harbor", + Version: "0.1.0", + }, + Capabilities: []*v1.ScannerCapability{{ + ConsumesMimeTypes: []string{ + v1.MimeTypeOCIArtifact, + v1.MimeTypeDockerArtifact, + }, + ProducesMimeTypes: []string{ + v1.MimeTypeNativeReport, + v1.MimeTypeRawReport, + }, + }}, + Properties: v1.ScannerProperties{ + "extra": "testing", + }, } suite.sample = &scanner.Registration{ @@ -57,6 +75,17 @@ func (suite *ControllerTestSuite) SetupSuite() { Description: "sample registration", URL: "https://sample.scanner.com", } + + mc := &MockClient{} + mc.On("GetMetadata").Return(m, nil) + + mcp := &MockClientPool{} + mcp.On("Get", suite.sample).Return(mc, nil) + suite.c = &basicController{ + manager: suite.mMgr, + proMetaMgr: suite.mMeta, + clientPool: mcp, + } } // Clear test case @@ -282,3 +311,50 @@ func (m *MockProMetaManager) List(name, value string) ([]*models.ProjectMetadata args := m.Called(name, value) return args.Get(0).([]*models.ProjectMetadata), args.Error(1) } + +// MockClientPool is defined and referred by other UT cases. +type MockClientPool struct { + mock.Mock +} + +// Get client +func (mcp *MockClientPool) Get(r *scanner.Registration) (v1.Client, error) { + args := mcp.Called(r) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(v1.Client), args.Error(1) +} + +// MockClient is defined and referred in other UT cases. +type MockClient struct { + mock.Mock +} + +// GetMetadata ... +func (mc *MockClient) GetMetadata() (*v1.ScannerAdapterMetadata, error) { + args := mc.Called() + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*v1.ScannerAdapterMetadata), args.Error(1) +} + +// SubmitScan ... +func (mc *MockClient) SubmitScan(req *v1.ScanRequest) (*v1.ScanResponse, error) { + args := mc.Called(req) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*v1.ScanResponse), args.Error(1) +} + +// GetScanReport ... +func (mc *MockClient) GetScanReport(scanRequestID, reportMIMEType string) (string, error) { + args := mc.Called(scanRequestID, reportMIMEType) + + return args.String(0), args.Error(1) +} diff --git a/src/pkg/scan/api/scanner/controller.go b/src/pkg/scan/api/scanner/controller.go index 048d94e960..d87928ca04 100644 --- a/src/pkg/scan/api/scanner/controller.go +++ b/src/pkg/scan/api/scanner/controller.go @@ -17,6 +17,7 @@ package scanner import ( "github.com/goharbor/harbor/src/pkg/q" "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" ) // Controller provides the related operations of scanner for the upper API. @@ -112,4 +113,26 @@ type Controller interface { // *scanner.Registration : the default scanner registration // error : non nil error if any errors occurred GetRegistrationByProject(projectID int64) (*scanner.Registration, error) + + // Ping pings Scanner Adapter to test EndpointURL and Authorization settings. + // The implementation is supposed to call the GetMetadata method on scanner.Client. + // Returns `nil` if connection succeeded, a non `nil` error otherwise. + // + // Arguments: + // registration *scanner.Registration : scanner registration to ping + // + // Returns: + // *v1.ScannerAdapterMetadata : metadata returned by the scanner if successfully ping + // error : non nil error if any errors occurred + Ping(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) + + // GetMetadata returns the metadata of the given scanner. + // + // Arguments: + // registrationUUID string : the UUID of the given scanner which is marked as default + // + // Returns: + // *v1.ScannerAdapterMetadata : metadata returned by the scanner if successfully ping + // error : non nil error if any errors occurred + GetMetadata(registrationUUID string) (*v1.ScannerAdapterMetadata, error) } diff --git a/src/pkg/scan/dao/scan/model.go b/src/pkg/scan/dao/scan/model.go index 5789e632c4..9d7f3ff4a2 100644 --- a/src/pkg/scan/dao/scan/model.go +++ b/src/pkg/scan/dao/scan/model.go @@ -25,6 +25,7 @@ type Report struct { RegistrationUUID string `orm:"column(registration_uuid)"` MimeType string `orm:"column(mime_type)"` JobID string `orm:"column(job_id)"` + TrackID string `orm:"column(track_id)"` Status string `orm:"column(status)"` StatusCode int `orm:"column(status_code)"` StatusRevision int64 `orm:"column(status_rev)"` diff --git a/src/pkg/scan/dao/scan/report.go b/src/pkg/scan/dao/scan/report.go index 653f864df6..6b428f8a2e 100644 --- a/src/pkg/scan/dao/scan/report.go +++ b/src/pkg/scan/dao/scan/report.go @@ -16,6 +16,7 @@ package scan import ( "fmt" + "time" "github.com/astaxie/beego/orm" "github.com/goharbor/harbor/src/common/dao" @@ -103,7 +104,7 @@ func UpdateReportData(uuid string, report string, statusRev int64) error { } // UpdateReportStatus updates the report `status` with conditions matched. -func UpdateReportStatus(uuid string, status string, statusCode int, statusRev int64) error { +func UpdateReportStatus(trackID string, status string, statusCode int, statusRev int64) error { o := dao.GetOrmer() qt := o.QueryTable(new(Report)) @@ -112,7 +113,13 @@ func UpdateReportStatus(uuid string, status string, statusCode int, statusRev in data["status_code"] = statusCode data["status_rev"] = statusRev - count, err := qt.Filter("uuid", uuid). + // Technically it is not correct, just to avoid changing interface and adding more code. + // running==2 + if statusCode > 2 { + data["end_time"] = time.Now().UTC() + } + + count, err := qt.Filter("track_id", trackID). Filter("status_rev__lte", statusRev). Filter("status_code__lte", statusCode).Update(data) @@ -121,20 +128,20 @@ func UpdateReportStatus(uuid string, status string, statusCode int, statusRev in } if count == 0 { - return errors.Errorf("no report with uuid %s updated", uuid) + return errors.Errorf("no report with track_id %s updated", trackID) } return nil } // UpdateJobID updates the report `job_id` column -func UpdateJobID(uuid string, jobID string) error { +func UpdateJobID(trackID string, jobID string) error { o := dao.GetOrmer() qt := o.QueryTable(new(Report)) params := make(orm.Params, 1) params["job_id"] = jobID - _, err := qt.Filter("uuid", uuid).Update(params) + _, err := qt.Filter("track_id", trackID).Update(params) return err } diff --git a/src/pkg/scan/dao/scan/report_test.go b/src/pkg/scan/dao/scan/report_test.go index 63d3181286..f2193b9bce 100644 --- a/src/pkg/scan/dao/scan/report_test.go +++ b/src/pkg/scan/dao/scan/report_test.go @@ -45,6 +45,7 @@ func (suite *ReportTestSuite) SetupSuite() { func (suite *ReportTestSuite) SetupTest() { r := &Report{ UUID: "uuid", + TrackID: "track-uuid", Digest: "digest1001", RegistrationUUID: "ruuid", MimeType: v1.MimeTypeNativeReport, @@ -95,7 +96,7 @@ func (suite *ReportTestSuite) TestReportList() { // TestReportUpdateJobID tests update job ID of the report. func (suite *ReportTestSuite) TestReportUpdateJobID() { - err := UpdateJobID("uuid", "jobid001") + err := UpdateJobID("track-uuid", "jobid001") require.NoError(suite.T(), err) l, err := ListReports(nil) @@ -120,12 +121,12 @@ func (suite *ReportTestSuite) TestReportUpdateReportData() { // TestReportUpdateStatus tests update the report status. func (suite *ReportTestSuite) TestReportUpdateStatus() { - err := UpdateReportStatus("uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 1000) + err := UpdateReportStatus("track-uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 1000) require.NoError(suite.T(), err) - err = UpdateReportStatus("uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 900) + err = UpdateReportStatus("track-uuid", job.RunningStatus.String(), job.RunningStatus.Code(), 900) require.Error(suite.T(), err) - err = UpdateReportStatus("uuid", job.PendingStatus.String(), job.PendingStatus.Code(), 1000) + err = UpdateReportStatus("track-uuid", job.PendingStatus.String(), job.PendingStatus.Code(), 1000) require.Error(suite.T(), err) } diff --git a/src/pkg/scan/dao/scanner/model.go b/src/pkg/scan/dao/scanner/model.go index bf711a7f33..8186aa0078 100644 --- a/src/pkg/scan/dao/scanner/model.go +++ b/src/pkg/scan/dao/scanner/model.go @@ -20,6 +20,8 @@ import ( "strings" "time" + "github.com/goharbor/harbor/src/pkg/scan/rest/auth" + "github.com/pkg/errors" ) @@ -45,6 +47,11 @@ type Registration struct { // Http connection settings SkipCertVerify bool `orm:"column(skip_cert_verify);default(false)" json:"skip_certVerify"` + // Extra info about the scanner + Scanner string `orm:"-" json:"scanner,omitempty"` + Vendor string `orm:"-" json:"vendor,omitempty"` + Version string `orm:"-" json:"version,omitempty"` + // Timestamps CreateTime time.Time `orm:"column(create_time);auto_now_add;type(datetime)" json:"create_time"` UpdateTime time.Time `orm:"column(update_time);auto_now;type(datetime)" json:"update_time"` @@ -89,6 +96,17 @@ func (r *Registration) Validate(checkUUID bool) error { return errors.Wrap(err, "scanner registration validate") } + if len(r.Auth) > 0 && + r.Auth != auth.Basic && + r.Auth != auth.Bearer && + r.Auth != auth.APIKey { + return errors.Errorf("auth type %s is not supported", r.Auth) + } + + if len(r.Auth) > 0 && len(r.AccessCredential) == 0 { + return errors.Errorf("access_credential is required for auth type %s", r.Auth) + } + return nil } diff --git a/src/pkg/scan/dao/scanner/registration.go b/src/pkg/scan/dao/scanner/registration.go index da2912dea6..db489fe174 100644 --- a/src/pkg/scan/dao/scanner/registration.go +++ b/src/pkg/scan/dao/scanner/registration.go @@ -16,6 +16,7 @@ package scanner import ( "fmt" + "strings" "github.com/astaxie/beego/orm" "github.com/goharbor/harbor/src/common/dao" @@ -93,6 +94,12 @@ func ListRegistrations(query *q.Query) ([]*Registration, error) { if query != nil { if len(query.Keywords) > 0 { for k, v := range query.Keywords { + if strings.HasPrefix(k, "ex_") { + kk := strings.TrimPrefix(k, "ex_") + qt = qt.Filter(kk, v) + continue + } + qt = qt.Filter(fmt.Sprintf("%s__icontains", k), v) } } @@ -111,20 +118,38 @@ func ListRegistrations(query *q.Query) ([]*Registration, error) { // SetDefaultRegistration sets the specified registration as default one func SetDefaultRegistration(UUID string) error { o := dao.GetOrmer() - qt := o.QueryTable(new(Registration)) - - _, err := qt.Filter("is_default", true).Update(orm.Params{ - "is_default": false, - }) - + err := o.Begin() if err != nil { return err } - qt2 := o.QueryTable(new(Registration)) - _, err = qt2.Filter("uuid", UUID).Update(orm.Params{ - "is_default": true, - }) + var count int64 + qt := o.QueryTable(new(Registration)) + count, err = qt.Filter("uuid", UUID). + Filter("disabled", false). + Update(orm.Params{ + "is_default": true, + }) + if err == nil && count == 0 { + err = errors.Errorf("set default for %s failed", UUID) + } + + if err == nil { + qt2 := o.QueryTable(new(Registration)) + _, err = qt2.Exclude("uuid__exact", UUID). + Filter("is_default", true). + Update(orm.Params{ + "is_default": false, + }) + } + + if err != nil { + if e := o.Rollback(); e != nil { + err = errors.Wrap(e, err.Error()) + } + } else { + err = o.Commit() + } return err } diff --git a/src/pkg/scan/dao/scanner/registration_test.go b/src/pkg/scan/dao/scanner/registration_test.go index d7a228a5f3..b44c4a4358 100644 --- a/src/pkg/scan/dao/scanner/registration_test.go +++ b/src/pkg/scan/dao/scanner/registration_test.go @@ -124,6 +124,22 @@ func (suite *RegistrationDAOTestSuite) TestList() { }) require.NoError(suite.T(), err) require.Equal(suite.T(), 0, len(l)) + + // Exact match + exactKeywords := make(map[string]interface{}) + exactKeywords["ex_name"] = "forUT" + l, err = ListRegistrations(&q.Query{ + Keywords: exactKeywords, + }) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 1, len(l)) + + exactKeywords["ex_name"] = "forU" + l, err = ListRegistrations(&q.Query{ + Keywords: exactKeywords, + }) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 0, len(l)) } // TestDefault tests set/get default @@ -138,4 +154,11 @@ func (suite *RegistrationDAOTestSuite) TestDefault() { dr, err = GetDefaultRegistration() require.NoError(suite.T(), err) require.NotNil(suite.T(), dr) + + dr.Disabled = true + err = UpdateRegistration(dr, "disabled") + require.NoError(suite.T(), err) + + err = SetDefaultRegistration(suite.registrationID) + require.Error(suite.T(), err) } diff --git a/src/pkg/scan/job.go b/src/pkg/scan/job.go index aa99199774..f8d10c104b 100644 --- a/src/pkg/scan/job.go +++ b/src/pkg/scan/job.go @@ -116,17 +116,18 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error { // Print related infos to log printJSONParameter(JobParamRegistration, params[JobParamRegistration].(string), myLogger) - printJSONParameter(JobParameterRequest, params[JobParameterRequest].(string), myLogger) + printJSONParameter(JobParameterRequest, removeAuthInfo(req), myLogger) + myLogger.Infof("Report mime types: %v\n", mimes) // Submit scan request to the scanner adapter client, err := v1.DefaultClientPool.Get(r) if err != nil { - return errors.Wrap(err, "run scan job") + return logAndWrapError(myLogger, err, "scan job: get client") } resp, err := client.SubmitScan(req) if err != nil { - return errors.Wrap(err, "run scan job") + return logAndWrapError(myLogger, err, "scan job: submit scan request") } // For collecting errors @@ -229,6 +230,13 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error { return err } +func logAndWrapError(logger logger.Interface, err error, message string) error { + e := errors.Wrap(err, message) + logger.Error(e) + + return e +} + func printJSONParameter(parameter string, v string, logger logger.Interface) { logger.Infof("%s:\n", parameter) printPrettyJSON([]byte(v), logger) @@ -244,6 +252,23 @@ func printPrettyJSON(in []byte, logger logger.Interface) { logger.Infof("%s\n", out.String()) } +func removeAuthInfo(sr *v1.ScanRequest) string { + req := &v1.ScanRequest{ + Artifact: sr.Artifact, + Registry: &v1.Registry{ + URL: sr.Registry.URL, + Authorization: "[HIDDEN]", + }, + } + + str, err := req.ToJSON() + if err != nil { + logger.Error(errors.Wrap(err, "scan job: remove auth")) + } + + return str +} + func extractScanReq(params job.Parameters) (*v1.ScanRequest, error) { v, ok := params[JobParameterRequest] if !ok { @@ -263,7 +288,6 @@ func extractScanReq(params job.Parameters) (*v1.ScanRequest, error) { if err := req.FromJSON(jsonData); err != nil { return nil, err } - if err := req.Validate(); err != nil { return nil, err } @@ -304,14 +328,24 @@ func extractMimeTypes(params job.Parameters) ([]string, error) { return nil, errors.Errorf("missing job parameter '%s'", JobParameterMimes) } - l, ok := v.([]string) + l, ok := v.([]interface{}) if !ok { return nil, errors.Errorf( - "malformed job parameter '%s', expecting string but got %s", + "malformed job parameter '%s', expecting []interface{} but got %s", JobParameterMimes, reflect.TypeOf(v).String(), ) } - return l, nil + mimes := make([]string, 0) + for _, v := range l { + mime, ok := v.(string) + if !ok { + return nil, errors.Errorf("expect string but got %s", reflect.TypeOf(v).String()) + } + + mimes = append(mimes, mime) + } + + return mimes, nil } diff --git a/src/pkg/scan/report/base_manager.go b/src/pkg/scan/report/base_manager.go index 163e2707fc..b4645eed74 100644 --- a/src/pkg/scan/report/base_manager.go +++ b/src/pkg/scan/report/base_manager.go @@ -57,15 +57,21 @@ func (bm *basicManager) Create(r *scan.Report) (string, error) { }) if err != nil { - return "", errors.Wrap(err, "check existence of report") + return "", errors.Wrap(err, "create report: check existence of report") } // Delete existing copy if len(existingCopies) > 0 { theCopy := existingCopies[0] - // Status conflict theStatus := job.Status(theCopy.Status) + // Status is an error message + if theStatus.Code() == -1 && theCopy.StatusCode == job.ErrorStatus.Code() { + // Mark as regular error status + theStatus = job.ErrorStatus + } + + // Status conflict if theStatus.Compare(job.RunningStatus) <= 0 { return "", errors.Errorf("conflict: a previous scanning is %s", theCopy.Status) } @@ -73,7 +79,7 @@ func (bm *basicManager) Create(r *scan.Report) (string, error) { // Otherwise it will be a completed report // Clear it before insert this new one if err := scan.DeleteReport(theCopy.UUID); err != nil { - return "", errors.Wrap(err, "clear old scan report") + return "", errors.Wrap(err, "create report: clear old scan report") } } @@ -91,12 +97,39 @@ func (bm *basicManager) Create(r *scan.Report) (string, error) { // Insert if _, err = scan.CreateReport(r); err != nil { - return "", errors.Wrap(err, "create report") + return "", errors.Wrap(err, "create report: insert") } return r.UUID, nil } +// Get ... +func (bm *basicManager) Get(uuid string) (*scan.Report, error) { + if len(uuid) == 0 { + return nil, errors.New("empty uuid to get scan report") + } + + kws := make(map[string]interface{}) + kws["uuid"] = uuid + + l, err := scan.ListReports(&q.Query{ + PageNumber: 1, + PageSize: 1, + Keywords: kws, + }) + + if err != nil { + return nil, errors.Wrap(err, "report manager: get") + } + + if len(l) == 0 { + // Not found + return nil, nil + } + + return l[0], nil +} + // GetBy ... func (bm *basicManager) GetBy(digest string, registrationUUID string, mimeTypes []string) ([]*scan.Report, error) { if len(digest) == 0 { @@ -121,24 +154,20 @@ func (bm *basicManager) GetBy(digest string, registrationUUID string, mimeTypes } // UpdateScanJobID ... -func (bm *basicManager) UpdateScanJobID(uuid string, jobID string) error { - if len(uuid) == 0 || len(jobID) == 0 { +func (bm *basicManager) UpdateScanJobID(trackID string, jobID string) error { + if len(trackID) == 0 || len(jobID) == 0 { return errors.New("bad arguments") } - return scan.UpdateJobID(uuid, jobID) + return scan.UpdateJobID(trackID, jobID) } // UpdateStatus ... -func (bm *basicManager) UpdateStatus(uuid string, status string, rev int64) error { - if len(uuid) == 0 { +func (bm *basicManager) UpdateStatus(trackID string, status string, rev int64) error { + if len(trackID) == 0 { return errors.New("missing uuid") } - if rev <= 0 { - return errors.New("invalid data revision") - } - stCode := job.ErrorStatus.Code() st := job.Status(status) // Check if it is job valid status. @@ -148,7 +177,7 @@ func (bm *basicManager) UpdateStatus(uuid string, status string, rev int64) erro stCode = st.Code() } - return scan.UpdateReportStatus(uuid, status, stCode, rev) + return scan.UpdateReportStatus(trackID, status, stCode, rev) } // UpdateReportData ... @@ -157,10 +186,6 @@ func (bm *basicManager) UpdateReportData(uuid string, report string, rev int64) return errors.New("missing uuid") } - if rev <= 0 { - return errors.New("invalid data revision") - } - if len(report) == 0 { return errors.New("missing report JSON data") } diff --git a/src/pkg/scan/report/base_manager_test.go b/src/pkg/scan/report/base_manager_test.go index e4b881e0a7..17d5fef1d2 100644 --- a/src/pkg/scan/report/base_manager_test.go +++ b/src/pkg/scan/report/base_manager_test.go @@ -52,6 +52,7 @@ func (suite *TestManagerSuite) SetupTest() { Digest: "d1000", RegistrationUUID: "ruuid", MimeType: v1.MimeTypeNativeReport, + TrackID: "tid001", } uuid, err := suite.m.Create(rp) @@ -70,13 +71,14 @@ func (suite *TestManagerSuite) TearDownTest() { // TestManagerCreateWithExisting tests the case that a copy already is there when creating report. func (suite *TestManagerSuite) TestManagerCreateWithExisting() { - err := suite.m.UpdateStatus(suite.rpUUID, job.SuccessStatus.String(), 2000) + err := suite.m.UpdateStatus("tid001", job.SuccessStatus.String(), 2000) require.NoError(suite.T(), err) rp := &scan.Report{ Digest: "d1000", RegistrationUUID: "ruuid", MimeType: v1.MimeTypeNativeReport, + TrackID: "tid002", } uuid, err := suite.m.Create(rp) @@ -87,6 +89,16 @@ func (suite *TestManagerSuite) TestManagerCreateWithExisting() { suite.rpUUID = uuid } +// TestManagerGet tests the get method. +func (suite *TestManagerSuite) TestManagerGet() { + sr, err := suite.m.Get(suite.rpUUID) + + require.NoError(suite.T(), err) + require.NotNil(suite.T(), sr) + + assert.Equal(suite.T(), "d1000", sr.Digest) +} + // TestManagerGetBy tests the get by method. func (suite *TestManagerSuite) TestManagerGetBy() { l, err := suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport}) @@ -113,7 +125,7 @@ func (suite *TestManagerSuite) TestManagerUpdateJobID() { oldJID := l[0].JobID - err = suite.m.UpdateScanJobID(suite.rpUUID, "jID1001") + err = suite.m.UpdateScanJobID("tid001", "jID1001") require.NoError(suite.T(), err) l, err = suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport}) @@ -132,7 +144,7 @@ func (suite *TestManagerSuite) TestManagerUpdateStatus() { oldSt := l[0].Status - err = suite.m.UpdateStatus(suite.rpUUID, job.SuccessStatus.String(), 10000) + err = suite.m.UpdateStatus("tid001", job.SuccessStatus.String(), 10000) require.NoError(suite.T(), err) l, err = suite.m.GetBy("d1000", "ruuid", []string{v1.MimeTypeNativeReport}) diff --git a/src/pkg/scan/report/manager.go b/src/pkg/scan/report/manager.go index 4c4ca13a1c..f4059abcc7 100644 --- a/src/pkg/scan/report/manager.go +++ b/src/pkg/scan/report/manager.go @@ -32,32 +32,32 @@ type Manager interface { // Update the scan job ID of the given report. // // Arguments: - // uuid string : uuid to identify the report - // jobID string: scan job ID + // trackID string : uuid to identify the report + // jobID string : scan job ID // // Returns: // error : non nil error if any errors occurred // - UpdateScanJobID(uuid string, jobID string) error + UpdateScanJobID(trackID string, jobID string) error // Update the status (mapping to the scan job status) of the given report. // // Arguments: - // uuid string : uuid to identify the report - // status string: status info - // rev int64 : data revision info + // trackID string : uuid to identify the report + // status string : status info + // rev int64 : data revision info // // Returns: // error : non nil error if any errors occurred // - UpdateStatus(uuid string, status string, rev int64) error + UpdateStatus(trackID string, status string, rev int64) error // Update the report data (with JSON format) of the given report. // // Arguments: - // uuid string : uuid to identify the report - // report string: report JSON data - // rev int64 : data revision info + // uuid string : uuid to identify the report + // report string : report JSON data + // rev int64 : data revision info // // Returns: // error : non nil error if any errors occurred @@ -77,4 +77,14 @@ type Manager interface { // []*scan.Report : report list // error : non nil error if any errors occurred GetBy(digest string, registrationUUID string, mimeTypes []string) ([]*scan.Report, error) + + // Get the report for the given uuid. + // + // Arguments: + // uuid string : uuid of the scan report + // + // Returns: + // *scan.Report : scan report + // error : non nil error if any errors occurred + Get(uuid string) (*scan.Report, error) } diff --git a/src/pkg/scan/report/summary.go b/src/pkg/scan/report/summary.go new file mode 100644 index 0000000000..5c5e37d571 --- /dev/null +++ b/src/pkg/scan/report/summary.go @@ -0,0 +1,97 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "reflect" + + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/pkg/scan/dao/scan" + v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/goharbor/harbor/src/pkg/scan/vuln" + "github.com/pkg/errors" +) + +// SupportedGenerators declares mappings between mime type and summary generator func. +var SupportedGenerators = map[string]SummaryGenerator{ + v1.MimeTypeNativeReport: GenerateNativeSummary, +} + +// GenerateSummary is a helper function to generate report +// summary based on the given report. +func GenerateSummary(r *scan.Report) (interface{}, error) { + g, ok := SupportedGenerators[r.MimeType] + if !ok { + return nil, errors.Errorf("no generator bound with mime type %s", r.MimeType) + } + + return g(r) +} + +// SummaryGenerator is a func template which used to generated report +// summary for relevant mime type. +type SummaryGenerator func(r *scan.Report) (interface{}, error) + +// GenerateNativeSummary generates the report summary for the native report. +func GenerateNativeSummary(r *scan.Report) (interface{}, error) { + sum := &vuln.NativeReportSummary{} + sum.ReportID = r.UUID + sum.StartTime = r.StartTime + sum.EndTime = r.EndTime + sum.Duration = r.EndTime.Unix() - r.EndTime.Unix() + + sum.ScanStatus = job.ErrorStatus.String() + if job.Status(r.Status).Code() != -1 { + sum.ScanStatus = r.Status + } + + // If the status is not success/stopped, there will not be any report. + if r.Status != job.SuccessStatus.String() && + r.Status != job.StoppedStatus.String() { + return sum, nil + } + + // Probably no report data if the job is interrupted + if len(r.Report) == 0 { + return nil, errors.Errorf("no report data for %s, status is: %s", r.UUID, sum.ScanStatus) + } + + raw, err := ResolveData(r.MimeType, []byte(r.Report)) + if err != nil { + return nil, err + } + + rp, ok := raw.(*vuln.Report) + if !ok { + return nil, errors.Errorf("type mismatch: expect *vuln.Report but got %s", reflect.TypeOf(raw).String()) + } + + sum.Severity = rp.Severity + vsum := &vuln.VulnerabilitySummary{ + Total: len(rp.Vulnerabilities), + Summary: make(vuln.SeveritySummary), + } + + for _, v := range rp.Vulnerabilities { + if num, ok := vsum.Summary[v.Severity]; ok { + vsum.Summary[v.Severity] = num + 1 + } else { + vsum.Summary[v.Severity] = 1 + } + } + sum.Summary = vsum + + return sum, nil +} diff --git a/src/pkg/scan/report/supported_mime_test.go b/src/pkg/scan/report/supported_mime_test.go index f118e25ff5..c4c167e0a0 100644 --- a/src/pkg/scan/report/supported_mime_test.go +++ b/src/pkg/scan/report/supported_mime_test.go @@ -69,6 +69,7 @@ func (suite *SupportedMimesSuite) SetupSuite() { func (suite *SupportedMimesSuite) TestResolveData() { obj, err := ResolveData(v1.MimeTypeNativeReport, suite.mockData) require.NoError(suite.T(), err) + require.NotNil(suite.T(), obj) require.Condition(suite.T(), func() (success bool) { rp, ok := obj.(*vuln.Report) success = ok && rp != nil && rp.Severity == vuln.High diff --git a/src/pkg/scan/report/supported_mimes.go b/src/pkg/scan/report/supported_mimes.go index 80bf60acaf..fc09aed651 100644 --- a/src/pkg/scan/report/supported_mimes.go +++ b/src/pkg/scan/report/supported_mimes.go @@ -31,13 +31,15 @@ var SupportedMimes = map[string]interface{}{ // ResolveData is a helper func to parse the JSON data with the given mime type. func ResolveData(mime string, jsonData []byte) (interface{}, error) { - if len(jsonData) == 0 { - return nil, errors.New("empty JSON data") - } - + // If no resolver defined for the given mime types, directly ignore it. + // The raw data will be used. t, ok := SupportedMimes[mime] if !ok { - return nil, errors.Errorf("report with mime type %s is not supported", mime) + return nil, nil + } + + if len(jsonData) == 0 { + return nil, errors.New("empty JSON data") } ty := reflect.TypeOf(t) diff --git a/src/pkg/scan/rest/v1/client.go b/src/pkg/scan/rest/v1/client.go index 9dfe9711df..a24eba3fb5 100644 --- a/src/pkg/scan/rest/v1/client.go +++ b/src/pkg/scan/rest/v1/client.go @@ -18,6 +18,7 @@ import ( "bytes" "crypto/tls" "encoding/json" + "fmt" "io/ioutil" "net" "net/http" @@ -157,7 +158,7 @@ func (c *basicClient) SubmitScan(req *ScanRequest) (*ScanResponse, error) { return nil, errors.Wrap(err, "v1 client: submit scan") } - respData, err := c.send(request, generalResponseHandler(http.StatusCreated)) + respData, err := c.send(request, generalResponseHandler(http.StatusAccepted)) if err != nil { return nil, errors.Wrap(err, "v1 client: submit scan") } @@ -199,7 +200,7 @@ func (c *basicClient) GetScanReport(scanRequestID, reportMIMEType string) (strin func (c *basicClient) send(req *http.Request, h responseHandler) ([]byte, error) { if c.authorizer != nil { if err := c.authorizer.Authorize(req); err != nil { - return nil, errors.Wrap(err, "authorization") + return nil, errors.Wrap(err, "send: authorization") } } @@ -266,12 +267,24 @@ func generalRespHandlerFunc(expectedCode, code int, resp *http.Response) ([]byte eResp := &ErrorResponse{ Err: &Error{}, } - if err := json.Unmarshal(buf, eResp); err == nil { - return nil, eResp + + err := json.Unmarshal(buf, eResp) + if err != nil { + return nil, errors.Wrap(err, "general response handler") } + + // Append more contexts + eResp.Err.Message = fmt.Sprintf( + "%s: general response handler: unexpected status code: %d, expected: %d", + eResp.Err.Message, + code, + expectedCode, + ) + + return nil, eResp } - return nil, errors.Errorf("unexpected status code: %d, response: %s", code, string(buf)) + return nil, errors.Errorf("general response handler: unexpected status code: %d, expected: %d", code, expectedCode) } return buf, nil diff --git a/src/pkg/scan/rest/v1/client_pool.go b/src/pkg/scan/rest/v1/client_pool.go index bf1dc3aa2b..6e45881010 100644 --- a/src/pkg/scan/rest/v1/client_pool.go +++ b/src/pkg/scan/rest/v1/client_pool.go @@ -102,7 +102,7 @@ func (bcp *basicClientPool) Get(r *scanner.Registration) (Client, error) { return nil, errors.New("nil scanner registration") } - if err := r.Validate(true); err != nil { + if err := r.Validate(false); err != nil { return nil, errors.Wrap(err, "client pool: get") } @@ -159,8 +159,7 @@ func (bcp *basicClientPool) deadCheck(key string, item *poolItem) { } func key(r *scanner.Registration) string { - return fmt.Sprintf("%s:%s:%s:%s:%v", - r.UUID, + return fmt.Sprintf("%s:%s:%s:%v", r.URL, r.Auth, r.AccessCredential, diff --git a/src/pkg/scan/rest/v1/client_test.go b/src/pkg/scan/rest/v1/client_test.go index 84767555b1..969189356d 100644 --- a/src/pkg/scan/rest/v1/client_test.go +++ b/src/pkg/scan/rest/v1/client_test.go @@ -115,7 +115,7 @@ type mockHandler struct{} // ServeHTTP ... func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch r.RequestURI { - case "/metadata": + case "/api/v1/metadata": if r.Method != http.MethodGet { w.WriteHeader(http.StatusForbidden) return @@ -126,7 +126,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { Vendor: "Harbor", Version: "0.1.0", }, - Capabilities: &ScannerCapability{ + Capabilities: []*ScannerCapability{{ ConsumesMimeTypes: []string{ MimeTypeOCIArtifact, MimeTypeDockerArtifact, @@ -135,7 +135,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { MimeTypeNativeReport, MimeTypeRawReport, }, - }, + }}, Properties: ScannerProperties{ "extra": "testing", }, @@ -144,7 +144,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) _, _ = w.Write(data) break - case "/scan": + case "/api/v1/scan": if r.Method != http.MethodPost { w.WriteHeader(http.StatusForbidden) return @@ -155,10 +155,10 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { data, _ := json.Marshal(res) - w.WriteHeader(http.StatusCreated) + w.WriteHeader(http.StatusAccepted) _, _ = w.Write(data) break - case "/scan/id1/report": + case "/api/v1/scan/id1/report": if r.Method != http.MethodGet { w.WriteHeader(http.StatusForbidden) return @@ -175,7 +175,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) _, _ = w.Write(data) break - case "/scan/id2/report": + case "/api/v1/scan/id2/report": if r.Method != http.MethodGet { w.WriteHeader(http.StatusForbidden) return @@ -183,7 +183,7 @@ func (mh *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("{}")) break - case "/scan/id3/report": + case "/api/v1/scan/id3/report": if r.Method != http.MethodGet { w.WriteHeader(http.StatusForbidden) return diff --git a/src/pkg/scan/rest/v1/models.go b/src/pkg/scan/rest/v1/models.go index 47ded4eb1b..cd817900dd 100644 --- a/src/pkg/scan/rest/v1/models.go +++ b/src/pkg/scan/rest/v1/models.go @@ -60,16 +60,21 @@ type ScannerProperties map[string]string // a scanner capable of scanning a given Artifact stored in its registry and making sure that it // can interpret a returned result. type ScannerAdapterMetadata struct { - Scanner *Scanner `json:"scanner"` - Capabilities *ScannerCapability `json:"capabilities"` - Properties ScannerProperties `json:"properties"` + Scanner *Scanner `json:"scanner"` + Capabilities []*ScannerCapability `json:"capabilities"` + Properties ScannerProperties `json:"properties"` } // Artifact represents an artifact stored in Registry. type Artifact struct { + // ID of the namespace (project). It will not be sent to scanner adapter. + NamespaceID int64 `json:"-"` // The full name of a Harbor repository containing the artifact, including the namespace. // For example, `library/oracle/nosql`. Repository string `json:"repository"` + // The info used to identify the version of the artifact, + // e.g: tag of image or version of the chart. + Tag string `json:"tag"` // The artifact's digest, consisting of an algorithm and hex portion. // For example, `sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b`, // represents sha256 based digest. diff --git a/src/pkg/scan/rest/v1/spec.go b/src/pkg/scan/rest/v1/spec.go index 6d4f6bf0ee..cf7ff8647e 100644 --- a/src/pkg/scan/rest/v1/spec.go +++ b/src/pkg/scan/rest/v1/spec.go @@ -39,6 +39,8 @@ const ( MimeTypeScanRequest = "application/vnd.scanner.adapter.scan.request+json; version=1.0" // MimeTypeScanResponse defines the mime type for scan response MimeTypeScanResponse = "application/vnd.scanner.adapter.scan.response+json; version=1.0" + + apiPrefix = "/api/v1" ) // RequestResolver is a function template to modify the API request, e.g: add headers @@ -70,6 +72,8 @@ func NewSpec(base string) *Spec { } } + s.baseRoute = fmt.Sprintf("%s%s", s.baseRoute, apiPrefix) + return s } diff --git a/src/pkg/scan/vuln/report.go b/src/pkg/scan/vuln/report.go index 7510f7cb68..57bbaf7d66 100644 --- a/src/pkg/scan/vuln/report.go +++ b/src/pkg/scan/vuln/report.go @@ -54,5 +54,5 @@ type VulnerabilityItem struct { // The list of link to the upstream database with the full description of the vulnerability. // Format: URI // e.g: List [ "https://security-tracker.debian.org/tracker/CVE-2017-8283" ] - Links []string + Links []string `json:"links"` } diff --git a/src/pkg/scan/vuln/summary.go b/src/pkg/scan/vuln/summary.go new file mode 100644 index 0000000000..27c596f735 --- /dev/null +++ b/src/pkg/scan/vuln/summary.go @@ -0,0 +1,41 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vuln + +import ( + "time" +) + +// NativeReportSummary is the default supported scan report summary model. +// Generated based on the report with v1.MimeTypeNativeReport mime type. +type NativeReportSummary struct { + ReportID string `json:"report_id"` + ScanStatus string `json:"scan_status"` + Severity Severity `json:"severity"` + Duration int64 `json:"duration"` + Summary *VulnerabilitySummary `json:"summary"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` +} + +// VulnerabilitySummary contains the total number of the found vulnerabilities number +// and numbers of each severity level. +type VulnerabilitySummary struct { + Total int `json:"total"` + Summary SeveritySummary `json:"summary"` +} + +// SeveritySummary ... +type SeveritySummary map[Severity]int diff --git a/src/replication/adapter/quayio/adapter.go b/src/replication/adapter/quayio/adapter.go new file mode 100644 index 0000000000..01dc5c5d83 --- /dev/null +++ b/src/replication/adapter/quayio/adapter.go @@ -0,0 +1,214 @@ +package quayio + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "strings" + + common_http "github.com/goharbor/harbor/src/common/http" + "github.com/goharbor/harbor/src/common/http/modifier" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/common/utils/registry/auth" + adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/adapter/native" + "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/replication/util" +) + +type adapter struct { + *native.Adapter + registry *model.Registry + client *common_http.Client +} + +func init() { + err := adp.RegisterFactory(model.RegistryTypeQuayio, func(registry *model.Registry) (adp.Adapter, error) { + return newAdapter(registry) + }) + if err != nil { + log.Errorf("failed to register factory for Quay.io: %v", err) + return + } + log.Infof("the factory of Quay.io adapter was registered") +} + +func newAdapter(registry *model.Registry) (*adapter, error) { + modifiers := []modifier.Modifier{ + &auth.UserAgentModifier{ + UserAgent: adp.UserAgentReplication, + }, + } + + var authorizer modifier.Modifier + if registry.Credential != nil && len(registry.Credential.AccessKey) != 0 { + authorizer = auth.NewAPIKeyAuthorizer("Authorization", fmt.Sprintf("Bearer %s", registry.Credential.AccessKey), auth.APIKeyInHeader) + } + + if authorizer != nil { + modifiers = append(modifiers, authorizer) + } + nativeRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer) + if err != nil { + return nil, err + } + + return &adapter{ + Adapter: nativeRegistryAdapter, + registry: registry, + client: common_http.NewClient( + &http.Client{ + Transport: util.GetHTTPTransport(registry.Insecure), + }, + modifiers..., + ), + }, nil +} + +// Info returns information of the registry +func (a *adapter) Info() (*model.RegistryInfo, error) { + return &model.RegistryInfo{ + Type: model.RegistryTypeQuayio, + SupportedResourceTypes: []model.ResourceType{ + model.ResourceTypeImage, + }, + SupportedResourceFilters: []*model.FilterStyle{ + { + Type: model.FilterTypeName, + Style: model.FilterStyleTypeText, + }, + { + Type: model.FilterTypeTag, + Style: model.FilterStyleTypeText, + }, + }, + SupportedTriggers: []model.TriggerType{ + model.TriggerTypeManual, + model.TriggerTypeScheduled, + }, + }, nil +} + +// HealthCheck checks health status of a registry +func (a *adapter) HealthCheck() (model.HealthStatus, error) { + err := a.PingSimple() + if err != nil { + return model.Unhealthy, nil + } + + return model.Healthy, nil +} + +// PrepareForPush does the prepare work that needed for pushing/uploading the resource +// eg: create the namespace or repository +func (a *adapter) PrepareForPush(resources []*model.Resource) error { + namespaces := []string{} + for _, resource := range resources { + if resource == nil { + return errors.New("the resource cannot be null") + } + if resource.Metadata == nil { + return errors.New("the metadata of resource cannot be null") + } + if resource.Metadata.Repository == nil { + return errors.New("the namespace of resource cannot be null") + } + if len(resource.Metadata.Repository.Name) == 0 { + return errors.New("the name of the namespace cannot be null") + } + paths := strings.Split(resource.Metadata.Repository.Name, "/") + namespace := paths[0] + namespaces = append(namespaces, namespace) + } + + for _, namespace := range namespaces { + err := a.createNamespace(&model.Namespace{ + Name: namespace, + }) + if err != nil { + return fmt.Errorf("create namespace '%s' in Quay.io error: %v", namespace, err) + } + log.Debugf("namespace %s created", namespace) + } + return nil +} + +// createNamespace creates a new namespace in Quay.io +func (a *adapter) createNamespace(namespace *model.Namespace) error { + ns, err := a.getNamespace(namespace.Name) + if err != nil { + return fmt.Errorf("check existence of namespace '%s' error: %v", namespace.Name, err) + } + + // If the namespace already exist, return succeeded directly. + if ns != nil { + log.Infof("Namespace %s already exist in Quay.io, skip it.", namespace.Name) + return nil + } + + org := &orgCreate{ + Name: namespace.Name, + Email: namespace.GetStringMetadata("email", namespace.Name), + } + b, err := json.Marshal(org) + if err != nil { + return err + } + + req, err := http.NewRequest(http.MethodPost, buildOrgURL(""), bytes.NewReader(b)) + if err != nil { + return err + } + + resp, err := a.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode/100 == 2 { + return nil + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + log.Errorf("create namespace error: %d -- %s", resp.StatusCode, string(body)) + return fmt.Errorf("%d -- %s", resp.StatusCode, body) +} + +// getNamespace get namespace from Quay.io, if the namespace not found, two nil would be returned. +func (a *adapter) getNamespace(namespace string) (*model.Namespace, error) { + req, err := http.NewRequest(http.MethodGet, buildOrgURL(namespace), nil) + if err != nil { + return nil, err + } + + resp, err := a.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } + + if resp.StatusCode/100 != 2 { + log.Errorf("get namespace error: %d -- %s", resp.StatusCode, string(body)) + return nil, fmt.Errorf("%d -- %s", resp.StatusCode, body) + } + + return &model.Namespace{ + Name: namespace, + }, nil +} diff --git a/src/replication/adapter/quayio/adapter_test.go b/src/replication/adapter/quayio/adapter_test.go new file mode 100644 index 0000000000..a77218c0b3 --- /dev/null +++ b/src/replication/adapter/quayio/adapter_test.go @@ -0,0 +1,48 @@ +package quayio + +import ( + "testing" + + adp "github.com/goharbor/harbor/src/replication/adapter" + "github.com/goharbor/harbor/src/replication/model" + "github.com/stretchr/testify/assert" +) + +func getMockAdapter(t *testing.T) adp.Adapter { + factory, _ := adp.GetFactory(model.RegistryTypeQuayio) + adapter, err := factory(&model.Registry{ + Type: model.RegistryTypeQuayio, + URL: "https://quay.io", + }) + assert.Nil(t, err) + return adapter +} +func TestAdapter_NewAdapter(t *testing.T) { + factory, err := adp.GetFactory("BadName") + assert.Nil(t, factory) + assert.NotNil(t, err) + + factory, err = adp.GetFactory(model.RegistryTypeQuayio) + assert.Nil(t, err) + assert.NotNil(t, factory) +} + +func TestAdapter_HealthCheck(t *testing.T) { + health, err := getMockAdapter(t).HealthCheck() + assert.Nil(t, err) + assert.Equal(t, string(health), model.Healthy) +} + +func TestAdapter_Info(t *testing.T) { + info, err := getMockAdapter(t).Info() + assert.Nil(t, err) + t.Log(info) +} + +func TestAdapter_PullManifests(t *testing.T) { + quayAdapter := getMockAdapter(t) + registry, _, err := quayAdapter.(*adapter).PullManifest("quay/busybox", "latest", []string{}) + assert.Nil(t, err) + assert.NotNil(t, registry) + t.Log(registry) +} diff --git a/src/replication/adapter/quayio/types.go b/src/replication/adapter/quayio/types.go new file mode 100644 index 0000000000..393dad0582 --- /dev/null +++ b/src/replication/adapter/quayio/types.go @@ -0,0 +1,12 @@ +package quayio + +import "fmt" + +type orgCreate struct { + Name string `json:"name"` + Email string `json:"email"` +} + +func buildOrgURL(orgName string) string { + return fmt.Sprintf("https://quay.io/api/v1/organization/%s", orgName) +} diff --git a/src/replication/model/registry.go b/src/replication/model/registry.go index a31f508968..3fa35c5aa0 100644 --- a/src/replication/model/registry.go +++ b/src/replication/model/registry.go @@ -31,6 +31,7 @@ const ( RegistryTypeAzureAcr RegistryType = "azure-acr" RegistryTypeAliAcr RegistryType = "ali-acr" RegistryTypeJfrogArtifactory RegistryType = "jfrog-artifactory" + RegistryTypeQuayio RegistryType = "quay-io" RegistryTypeHelmHub RegistryType = "helm-hub" diff --git a/src/replication/replication.go b/src/replication/replication.go index b06404874b..4eb0dde291 100644 --- a/src/replication/replication.go +++ b/src/replication/replication.go @@ -45,6 +45,8 @@ import ( _ "github.com/goharbor/harbor/src/replication/adapter/aliacr" // register the Jfrog Artifactory adapter _ "github.com/goharbor/harbor/src/replication/adapter/jfrog" + // register the Quay.io adapter + _ "github.com/goharbor/harbor/src/replication/adapter/quayio" // register the Helm Hub adapter _ "github.com/goharbor/harbor/src/replication/adapter/helmhub" ) diff --git a/tests/robot-cases/Group0-BAT/API_DB.robot b/tests/robot-cases/Group0-BAT/API_DB.robot index c2ee1d0717..f955e281e6 100644 --- a/tests/robot-cases/Group0-BAT/API_DB.robot +++ b/tests/robot-cases/Group0-BAT/API_DB.robot @@ -29,16 +29,18 @@ Test Case - Add Replication Rule Harbor API Test ./tests/apitests/python/test_add_replication_rule.py Test Case - Edit Project Creation Harbor API Test ./tests/apitests/python/test_edit_project_creation.py -Test Case - Scan Image - Harbor API Test ./tests/apitests/python/test_scan_image.py +*** Enable this case after deployment change PR merged *** +*** Test Case - Scan Image *** +*** Harbor API Test ./tests/apitests/python/test_scan_image.py *** Test Case - Manage Project Member Harbor API Test ./tests/apitests/python/test_manage_project_member.py Test Case - Project Level Policy Content Trust Harbor API Test ./tests/apitests/python/test_project_level_policy_content_trust.py Test Case - User View Logs Harbor API Test ./tests/apitests/python/test_user_view_logs.py -Test Case - Scan All Images - Harbor API Test ./tests/apitests/python/test_scan_all_images.py +*** Enable this case after deployment change PR merged *** +*** Test Case - Scan All Images *** +*** Harbor API Test ./tests/apitests/python/test_scan_all_images.py *** Test Case - List Helm Charts Harbor API Test ./tests/apitests/python/test_list_helm_charts.py Test Case - Assign Sys Admin