Merge pull request #11505 from heww/revert-registry-authorization-type-support

feat(scan): revert bearer token support for scanner
This commit is contained in:
He Weiwei 2020-04-13 11:19:02 +08:00 committed by GitHub
commit 0b87eaf039
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 308 additions and 49 deletions

View File

@ -0,0 +1,32 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"context"
)
type cfgMgrKey struct{}
// FromContext returns CfgManager from context
func FromContext(ctx context.Context) (*CfgManager, bool) {
m, ok := ctx.Value(cfgMgrKey{}).(*CfgManager)
return m, ok
}
// NewContext returns context with CfgManager
func NewContext(ctx context.Context, m *CfgManager) context.Context {
return context.WithValue(ctx, cfgMgrKey{}, m)
}

View File

@ -17,7 +17,6 @@ package scan
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"sync"
@ -28,7 +27,6 @@ import (
sc "github.com/goharbor/harbor/src/controller/scanner"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/permission/types"
@ -217,7 +215,7 @@ func (bc *basicController) Scan(ctx context.Context, artifact *ar.Artifact, opti
errs = errs[:0]
for _, param := range params {
if err := bc.scanArtifact(ctx, r, param.Artifact, param.TrackID, param.ProducesMimes); err != nil {
log.Warningf("scan artifact %s@%s failed, error: %v", artifact.RepositoryName, artifact.Digest, err)
log.G(ctx).Warningf("scan artifact %s@%s failed, error: %v", artifact.RepositoryName, artifact.Digest, err)
errs = append(errs, err)
}
}
@ -298,7 +296,7 @@ func (bc *basicController) scanArtifact(ctx context.Context, r *scanner.Registra
// Insert the generated job ID now
// It will not block the whole process. If any errors happened, just logged.
if err := bc.manager.UpdateScanJobID(trackID, jobID); err != nil {
logger.Error(errors.Wrap(err, "scan controller: scan"))
log.G(ctx).Error(errors.Wrap(err, "scan controller: scan"))
}
return nil
@ -508,14 +506,22 @@ func (bc *basicController) HandleJobHooks(trackID string, change *job.StatusChan
// Clear robot account
// Only when the job is successfully done!
if change.Status == job.SuccessStatus.String() {
if v, ok := change.Metadata.Parameters[sca.JobParameterRobotID]; ok {
if rid, y := v.(float64); y {
if err := robot.RobotCtr.DeleteRobotAccount(int64(rid)); err != nil {
// Should not block the main flow, just logged
if v, ok := change.Metadata.Parameters[sca.JobParameterRobot]; ok {
if jsonData, y := v.(string); y {
r := &model.Robot{}
if err := r.FromJSON(jsonData); err != nil {
log.Error(errors.Wrap(err, "scan controller: handle job hook"))
} else {
log.Debugf("Robot account with id %d for the scan %s is removed", int64(rid), trackID)
}
if r.ID > 0 {
if err := robot.RobotCtr.DeleteRobotAccount(r.ID); err != nil {
// Should not block the main flow, just logged
log.Error(errors.Wrap(err, "scan controller: handle job hook"))
} else {
log.Debugf("Robot account with id %d for the scan %s is removed", r.ID, trackID)
}
}
}
}
}
@ -617,14 +623,10 @@ func (bc *basicController) launchScanJob(trackID string, artifact *ar.Artifact,
return "", errors.Wrap(err, "scan controller: launch scan job")
}
basic := fmt.Sprintf("%s:%s", robot.Name, robot.Token)
authorization := fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(basic)))
// Set job parameters
scanReq := &v1.ScanRequest{
Registry: &v1.Registry{
URL: registryAddr,
Authorization: authorization,
URL: registryAddr,
},
Artifact: &v1.Artifact{
NamespaceID: artifact.ProjectID,
@ -644,11 +646,17 @@ func (bc *basicController) launchScanJob(trackID string, artifact *ar.Artifact,
return "", errors.Wrap(err, "launch scan job")
}
robotJSON, err := robot.ToJSON()
if err != nil {
return "", errors.Wrap(err, "launch scan job")
}
params := make(map[string]interface{})
params[sca.JobParamRegistration] = rJSON
params[sca.JobParameterAuthType] = registration.GetRegistryAuthorizationType()
params[sca.JobParameterRequest] = sJSON
params[sca.JobParameterMimes] = mimes
params[sca.JobParameterRobotID] = robot.ID
params[sca.JobParameterRobot] = robotJSON
// Launch job
callbackURL, err := bc.config(configCoreInternalAddr)

View File

@ -192,8 +192,7 @@ func (suite *ControllerTestSuite) SetupSuite() {
// Set job parameters
req := &v1.ScanRequest{
Registry: &v1.Registry{
URL: "https://core.com",
Authorization: "Basic " + base64.StdEncoding.EncodeToString([]byte(rname+":robot-account")),
URL: "https://core.com",
},
Artifact: &v1.Artifact{
NamespaceID: suite.artifact.ProjectID,
@ -209,12 +208,17 @@ func (suite *ControllerTestSuite) SetupSuite() {
regJSON, err := suite.registration.ToJSON()
require.NoError(suite.T(), err)
rb, _ := rc.CreateRobotAccount(account)
robotJSON, err := rb.ToJSON()
require.NoError(suite.T(), err)
jc := &MockJobServiceClient{}
params := make(map[string]interface{})
params[sca.JobParamRegistration] = regJSON
params[sca.JobParameterRequest] = rJSON
params[sca.JobParameterMimes] = []string{v1.MimeTypeNativeReport}
params[sca.JobParameterRobotID] = int64(1)
params[sca.JobParameterAuthType] = "Basic"
params[sca.JobParameterRobot] = robotJSON
j := &jm.JobData{
Name: job.ImageScanJob,

View File

@ -16,20 +16,20 @@ package impl
import (
"context"
"errors"
"fmt"
o "github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/lib/orm"
"math"
"sync"
"time"
"errors"
o "github.com/astaxie/beego/orm"
comcfg "github.com/goharbor/harbor/src/common/config"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/jobservice/config"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/jobservice/logger/sweeper"
"github.com/goharbor/harbor/src/lib/orm"
)
const (
@ -55,7 +55,7 @@ type Context struct {
// NewContext ...
func NewContext(sysCtx context.Context, cfgMgr *comcfg.CfgManager) *Context {
return &Context{
sysContext: sysCtx,
sysContext: comcfg.NewContext(sysCtx, cfgMgr),
cfgMgr: *cfgMgr,
properties: make(map[string]interface{}),
}

View File

@ -1,6 +1,8 @@
package model
import (
"encoding/json"
"errors"
"time"
"github.com/astaxie/beego/orm"
@ -35,6 +37,25 @@ func (r *Robot) TableName() string {
return RobotTable
}
// FromJSON parses robot from json data
func (r *Robot) FromJSON(jsonData string) error {
if len(jsonData) == 0 {
return errors.New("empty json data to parse")
}
return json.Unmarshal([]byte(jsonData), r)
}
// ToJSON marshals Robot to JSON data
func (r *Robot) ToJSON() (string, error) {
data, err := json.Marshal(r)
if err != nil {
return "", err
}
return string(data), nil
}
// RobotQuery ...
type RobotQuery struct {
Name string

View File

@ -25,6 +25,12 @@ import (
"github.com/pkg/errors"
)
const (
authorizationType = "harbor.scanner-adapter/registry-authorization-type"
authorizationBearer = "Bearer"
authorizationBasic = "Basic"
)
// Registration represents a named configuration for invoking a scanner via its adapter.
// UUID will be used to track the scanner.Endpoint as unique ID
type Registration struct {
@ -178,6 +184,22 @@ func (r *Registration) GetCapability(mimeType string) *v1.ScannerCapability {
return nil
}
// GetRegistryAuthorizationType returns the registry authorization type of the scanner
func (r *Registration) GetRegistryAuthorizationType() string {
var auth string
if r.Metadata != nil && r.Metadata.Properties != nil {
if v, ok := r.Metadata.Properties[authorizationType]; ok {
auth = v
}
}
if auth != authorizationBasic && auth != authorizationBearer {
auth = authorizationBasic
}
return auth
}
// Check the registration URL with url package
func checkURL(u string) error {
if len(strings.TrimSpace(u)) == 0 {

View File

@ -16,14 +16,23 @@ package scan
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"reflect"
"sync"
"time"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/config"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/robot/model"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/goharbor/harbor/src/pkg/scan/report"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
@ -37,11 +46,18 @@ const (
JobParameterRequest = "scanRequest"
// JobParameterMimes ...
JobParameterMimes = "mimeTypes"
// JobParameterRobotID ...
JobParameterRobotID = "robotID"
// JobParameterAuthType ...
JobParameterAuthType = "authType"
// JobParameterRobot ...
JobParameterRobot = "robotAccount"
checkTimeout = 30 * time.Minute
firstCheckInterval = 2 * time.Second
authorizationBearer = "Bearer"
authorizationBasic = "Basic"
service = "harbor-registry"
)
// CheckInReport defines model for checking in the scan report with specified mime.
@ -103,9 +119,18 @@ func (j *Job) Validate(params job.Parameters) error {
return errors.Wrap(err, "job validate")
}
// No need to check param robotID which os treated as an optional one.
// It is used to clear the generated robot account to reduce dirty data.
// Failure of doing this will not influence the main flow.
if _, err := extractRobotAccount(params); err != nil {
return errors.Wrap(err, "job validate")
}
authType, err := extractAuthType(params)
if err != nil {
return errors.Wrap(err, "job validate")
}
if authType != authorizationBearer && authType != authorizationBasic {
return errors.Wrapf(err, "job validate: not support auth type %s", authType)
}
return nil
}
@ -133,6 +158,25 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error {
// Ignore the namespace ID here
req.Artifact.NamespaceID = 0
robotAccount, _ := extractRobotAccount(params)
var authorization string
authType, _ := extractAuthType(params)
if authType == authorizationBearer {
tokenURL, err := getInternalTokenServiceEndpoint(ctx)
if err != nil {
return errors.Wrap(err, "scan job: get token service endpoint")
}
authorization, err = makeBearerAuthorization(robotAccount, tokenURL, req.Artifact.Repository)
} else {
authorization, err = makeBasicAuthorization(robotAccount)
}
if err != nil {
logAndWrapError(myLogger, err, "scan job: make authorization")
}
req.Registry.Authorization = authorization
resp, err := client.SubmitScan(req)
if err != nil {
return logAndWrapError(myLogger, err, "scan job: submit scan request")
@ -331,6 +375,29 @@ func extractRegistration(params job.Parameters) (*scanner.Registration, error) {
return r, nil
}
func extractRobotAccount(params job.Parameters) (*model.Robot, error) {
v, ok := params[JobParameterRobot]
if !ok {
return nil, errors.Errorf("missing job parameter '%s'", JobParameterRobot)
}
jsonData, ok := v.(string)
if !ok {
return nil, errors.Errorf(
"malformed job parameter '%s', expecting string but got %s",
JobParameterRobot,
reflect.TypeOf(v).String(),
)
}
r := &model.Robot{}
if err := r.FromJSON(jsonData); err != nil {
return nil, err
}
return r, nil
}
func extractMimeTypes(params job.Parameters) ([]string, error) {
v, ok := params[JobParameterMimes]
if !ok {
@ -358,3 +425,85 @@ func extractMimeTypes(params job.Parameters) ([]string, error) {
return mimes, nil
}
func extractAuthType(params job.Parameters) (string, error) {
v, ok := params[JobParameterAuthType]
if !ok {
return "", errors.Errorf("missing job parameter '%s'", JobParameterAuthType)
}
authType, ok := v.(string)
if !ok {
return "", errors.Errorf(
"malformed job parameter '%s', expecting string but got %s",
JobParameterAuthType,
reflect.TypeOf(v).String(),
)
}
return authType, nil
}
func getInternalTokenServiceEndpoint(ctx job.Context) (string, error) {
cfgMgr, ok := config.FromContext(ctx.SystemContext())
if !ok {
return "", errors.Errorf("failed to get config manager")
}
return cfgMgr.Get(common.CoreURL).GetString() + "/service/token", nil
}
// makeBasicAuthorization creates authorization from a robot account based on the arguments for scanning.
func makeBasicAuthorization(robotAccount *model.Robot) (string, error) {
basic := fmt.Sprintf("%s:%s", robotAccount.Name, robotAccount.Token)
encoded := base64.StdEncoding.EncodeToString([]byte(basic))
return fmt.Sprintf("Basic %s", encoded), nil
}
// makeBearerAuthorization creates bearer token from a robot account
func makeBearerAuthorization(robotAccount *model.Robot, tokenURL string, repository string) (string, error) {
u, err := url.Parse(tokenURL)
if err != nil {
return "", err
}
query := u.Query()
query.Add("service", service)
query.Add("scope", fmt.Sprintf("repository:%s:pull", repository))
u.RawQuery = query.Encode()
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return "", err
}
auth, _ := makeBasicAuthorization(robotAccount)
req.Header.Set("Authorization", auth)
client := &http.Client{
Transport: commonhttp.GetHTTPTransportByInsecure(true),
}
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("get bearer token failed, %s", string(data))
}
token := &models.Token{}
if err = json.Unmarshal(data, token); err != nil {
return "", err
}
return fmt.Sprintf("Bearer %s", token.GetToken()), nil
}

View File

@ -20,6 +20,7 @@ import (
"time"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/robot/model"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
@ -77,7 +78,7 @@ func (suite *JobTestSuite) TestJob() {
sr := &v1.ScanRequest{
Registry: &v1.Registry{
URL: "http://localhost:5000",
Authorization: "the_token",
Authorization: "Basic cm9ib3Q6dG9rZW4=",
},
Artifact: &v1.Artifact{
Repository: "library/test_job",
@ -89,12 +90,23 @@ func (suite *JobTestSuite) TestJob() {
sData, err := sr.ToJSON()
require.NoError(suite.T(), err)
robot := &model.Robot{
ID: 1,
Name: "robot",
Token: "token",
}
robotData, err := robot.ToJSON()
require.NoError(suite.T(), err)
mimeTypes := []string{v1.MimeTypeNativeReport}
jp := make(job.Parameters)
jp[JobParamRegistration] = rData
jp[JobParameterRequest] = sData
jp[JobParameterMimes] = mimeTypes
jp[JobParameterAuthType] = "Basic"
jp[JobParameterRobot] = robotData
mc := &v1testing.Client{}
sre := &v1.ScanResponse{

View File

@ -149,8 +149,7 @@ func (s *ScanRequest) ToJSON() (string, error) {
// Validate ScanRequest
func (s *ScanRequest) Validate() error {
if s.Registry == nil ||
len(s.Registry.URL) == 0 ||
len(s.Registry.Authorization) == 0 {
len(s.Registry.URL) == 0 {
return errors.New("scan request: invalid registry")
}

View File

@ -2,16 +2,16 @@ package contenttrust
import (
"fmt"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/security"
"net/http"
"github.com/goharbor/harbor/src/controller/artifact"
"github.com/goharbor/harbor/src/controller/project"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/signature"
"github.com/goharbor/harbor/src/server/middleware"
"net/http"
"github.com/goharbor/harbor/src/server/middleware/util"
)
var (
@ -30,6 +30,9 @@ var (
func Middleware() func(http.Handler) http.Handler {
return middleware.BeforeRequest(func(r *http.Request) error {
ctx := r.Context()
logger := log.G(ctx)
none := lib.ArtifactInfo{}
af := lib.GetArtifactInfo(ctx)
if af == none {
@ -46,11 +49,8 @@ func Middleware() func(http.Handler) http.Handler {
if err != nil {
return err
}
securityCtx, ok := security.FromContext(ctx)
// only authenticated robot account with scanner pull access can bypass.
if ok && securityCtx.IsAuthenticated() &&
(securityCtx.Name() == "robot" || securityCtx.Name() == "v2token") &&
securityCtx.Can(rbac.ActionScannerPull, rbac.NewProjectNamespace(pro.ProjectID).Resource(rbac.ResourceRepository)) {
if util.SkipPolicyChecking(ctx, pro.ProjectID) {
// the artifact is pulling by the scanner, skip the checking
logger.Debugf("artifact %s@%s is pulling by the scanner, skip the checking", af.Repository, af.Digest)
return nil

View File

@ -150,7 +150,7 @@ func (suite *MiddlewareTestSuite) TestScannerPulling() {
mock.OnAnything(suite.artifactController, "GetByReference").Return(suite.artifact, nil)
mock.OnAnything(suite.projectController, "GetByName").Return(suite.project, nil)
securityCtx := &securitytesting.Context{}
mock.OnAnything(securityCtx, "Name").Return("robot")
mock.OnAnything(securityCtx, "Name").Return("v2token")
mock.OnAnything(securityCtx, "Can").Return(true, nil)
mock.OnAnything(securityCtx, "IsAuthenticated").Return(true)

View File

@ -15,12 +15,15 @@
package util
import (
"context"
"fmt"
"net/http"
"path"
"strings"
"github.com/goharbor/harbor/src/common/api"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/security"
"github.com/goharbor/harbor/src/pkg/distribution"
)
@ -51,3 +54,16 @@ func ParseProjectName(r *http.Request) string {
return projectName
}
// SkipPolicyChecking ...
func SkipPolicyChecking(ctx context.Context, projectID int64) bool {
secCtx, ok := security.FromContext(ctx)
// only scanner pull access can bypass.
if ok && secCtx.Name() == "v2token" &&
secCtx.Can(rbac.ActionScannerPull, rbac.NewProjectNamespace(projectID).Resource(rbac.ResourceRepository)) {
return true
}
return false
}

View File

@ -18,8 +18,6 @@ import (
"fmt"
"net/http"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/security"
"github.com/goharbor/harbor/src/controller/artifact"
"github.com/goharbor/harbor/src/controller/project"
"github.com/goharbor/harbor/src/controller/scan"
@ -30,6 +28,7 @@ import (
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/goharbor/harbor/src/pkg/scan/vuln"
"github.com/goharbor/harbor/src/server/middleware"
"github.com/goharbor/harbor/src/server/middleware/util"
)
var (
@ -71,10 +70,7 @@ func Middleware() func(http.Handler) http.Handler {
return nil
}
securityCtx, ok := security.FromContext(ctx)
if ok &&
(securityCtx.Name() == "robot" || securityCtx.Name() == "v2token") &&
securityCtx.Can(rbac.ActionScannerPull, rbac.NewProjectNamespace(proj.ProjectID).Resource(rbac.ResourceRepository)) {
if util.SkipPolicyChecking(ctx, proj.ProjectID) {
// the artifact is pulling by the scanner, skip the checking
logger.Debugf("artifact %s@%s is pulling by the scanner, skip the checking", art.RepositoryName, art.Digest)
return nil

View File

@ -162,7 +162,7 @@ func (suite *MiddlewareTestSuite) TestPreventionDisabled() {
suite.Equal(rr.Code, http.StatusOK)
}
func (suite *MiddlewareTestSuite) TestNonRobotPulling() {
func (suite *MiddlewareTestSuite) TestNonScannerPulling() {
mock.OnAnything(suite.artifactController, "GetByReference").Return(suite.artifact, nil)
mock.OnAnything(suite.projectController, "Get").Return(suite.project, nil)
securityCtx := &securitytesting.Context{}
@ -182,7 +182,7 @@ func (suite *MiddlewareTestSuite) TestScannerPulling() {
mock.OnAnything(suite.artifactController, "GetByReference").Return(suite.artifact, nil)
mock.OnAnything(suite.projectController, "Get").Return(suite.project, nil)
securityCtx := &securitytesting.Context{}
mock.OnAnything(securityCtx, "Name").Return("robot")
mock.OnAnything(securityCtx, "Name").Return("v2token")
mock.OnAnything(securityCtx, "Can").Return(true, nil)
req := suite.makeRequest()