Add Scan All job to job service (#5934)

This commit adds the job to scan all images on registry.
It also makes necessary change to Secret based security context, to
job service has higher permission to call the API of core service.

Signed-off-by: Daniel Jiang <jiangd@vmware.com>

src/common/job/const.go

@@ -3,6 +3,8 @@ package job
 const (
 	// ImageScanJob is name of scan job it will be used as key to register to job service.
 	ImageScanJob = "IMAGE_SCAN"
+	// ImageScanAllJob is the name of "scanall" job in job service
+	ImageScanAllJob = "IMAGE_SCAN_ALL"
 	// ImageTransfer : the name of image transfer job in job service
 	ImageTransfer = "IMAGE_TRANSFER"
 	// ImageDelete : the name of image delete job in job service

src/common/security/secret/context.go

@@ -71,7 +71,7 @@ func (s *SecurityContext) IsSolutionUser() bool {
 }
 
 // HasReadPerm returns true if the corresponding user of the secret
-// is jobservice, otherwise returns false
+// is jobservice or core service, otherwise returns false
 func (s *SecurityContext) HasReadPerm(projectIDOrName interface{}) bool {
 	if s.store == nil {
 		return false
@@ -79,14 +79,22 @@ func (s *SecurityContext) HasReadPerm(projectIDOrName interface{}) bool {
 	return s.store.GetUsername(s.secret) == secret.JobserviceUser || s.store.GetUsername(s.secret) == secret.CoreUser
 }
 
-// HasWritePerm always returns false
+// HasWritePerm returns true if the corresponding user of the secret
+// is jobservice or core service, otherwise returns false
 func (s *SecurityContext) HasWritePerm(projectIDOrName interface{}) bool {
-	return false
+	if s.store == nil {
+		return false
+	}
+	return s.store.GetUsername(s.secret) == secret.JobserviceUser || s.store.GetUsername(s.secret) == secret.CoreUser
 }
 
-// HasAllPerm always returns false
+// HasAllPerm returns true if the corresponding user of the secret
+// is jobservice or core service, otherwise returns false
 func (s *SecurityContext) HasAllPerm(projectIDOrName interface{}) bool {
-	return false
+	if s.store == nil {
+		return false
+	}
+	return s.store.GetUsername(s.secret) == secret.JobserviceUser || s.store.GetUsername(s.secret) == secret.CoreUser
 }
 
 // GetMyProjects ...

src/jobservice/job/impl/scan/all.go

@@ -0,0 +1,141 @@
+// Copyright 2018 The Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scan
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/goharbor/harbor/src/common"
+	"github.com/goharbor/harbor/src/common/dao"
+	"github.com/goharbor/harbor/src/jobservice/env"
+	"github.com/goharbor/harbor/src/jobservice/job/impl/utils"
+)
+
+// All query the DB and Registry for all image and tags,
+// then call Harbor's API to scan each of them.
+type All struct {
+	registryURL          string
+	secret               string
+	tokenServiceEndpoint string
+	harborAPIEndpoint    string
+	coreClient           *http.Client
+}
+
+// MaxFails implements the interface in job/Interface
+func (sa *All) MaxFails() uint {
+	return 1
+}
+
+// ShouldRetry implements the interface in job/Interface
+func (sa *All) ShouldRetry() bool {
+	return false
+}
+
+// Validate implements the interface in job/Interface
+func (sa *All) Validate(params map[string]interface{}) error {
+	if len(params) > 0 {
+		return fmt.Errorf("the parms should be empty for scan all job")
+	}
+	return nil
+}
+
+// Run implements the interface in job/Interface
+func (sa *All) Run(ctx env.JobContext, params map[string]interface{}) error {
+	logger := ctx.GetLogger()
+	logger.Info("Scanning all the images in the registry")
+	err := sa.init(ctx)
+	if err != nil {
+		logger.Errorf("Failed to initialize the job handler, error: %v", err)
+		return err
+	}
+
+	repos, err := dao.GetRepositories()
+	if err != nil {
+		logger.Errorf("Failed to get the list of repositories, error: %v", err)
+		return err
+	}
+
+	for _, r := range repos {
+		repoClient, err := utils.NewRepositoryClientForJobservice(r.Name, sa.registryURL, sa.secret, sa.tokenServiceEndpoint)
+		if err != nil {
+			logger.Errorf("Failed to get repo client for repo: %s, error: %v", r.Name, err)
+			continue
+		}
+		tags, err := repoClient.ListTag()
+		if err != nil {
+			logger.Errorf("Failed to get tags for repo: %s, error: %v", r.Name, err)
+			continue
+		}
+		for _, t := range tags {
+			logger.Infof("Calling harbor-core API to scan image, %s:%s", r.Name, t)
+			resp, err := sa.coreClient.Post(fmt.Sprintf("%s/repositories/%s/tags/%s/scan", sa.harborAPIEndpoint, r.Name, t),
+				"application/json",
+				bytes.NewReader([]byte("{}")))
+			if err != nil {
+				logger.Errorf("Failed to trigger image scan, error: %v", err)
+			} else {
+				data, err := ioutil.ReadAll(resp.Body)
+				if err != nil {
+					logger.Errorf("Failed to read response, error: %v", err)
+				} else if resp.StatusCode != http.StatusOK {
+					logger.Errorf("Unexpected response code: %d, data: %v", resp.StatusCode, data)
+				}
+				resp.Body.Close()
+			}
+		}
+
+	}
+
+	return nil
+}
+
+func (sa *All) init(ctx env.JobContext) error {
+	if v, err := getAttrFromCtx(ctx, common.RegistryURL); err == nil {
+		sa.registryURL = v
+	} else {
+		return err
+	}
+	if v := os.Getenv("JOBSERVICE_SECRET"); len(v) > 0 {
+		sa.secret = v
+	} else {
+		return fmt.Errorf("failed to read evnironment variable JOBSERVICE_SECRET")
+	}
+	sa.coreClient, _ = utils.GetClient()
+	if v, err := getAttrFromCtx(ctx, common.TokenServiceURL); err == nil {
+		sa.tokenServiceEndpoint = v
+	} else {
+		return err
+	}
+	if v, err := getAttrFromCtx(ctx, common.CoreURL); err == nil {
+		v = strings.TrimSuffix(v, "/")
+		sa.harborAPIEndpoint = v + "/api"
+	} else {
+		return err
+	}
+	return nil
+}
+
+func getAttrFromCtx(ctx env.JobContext, key string) (string, error) {
+	if v, ok := ctx.Get(key); ok && len(v.(string)) > 0 {
+		return v.(string), nil
+	}
+	return "", fmt.Errorf("Failed to get required property: %s", key)
+}

src/jobservice/job/impl/utils/utils.go

@@ -3,6 +3,8 @@ package utils
 import (
 	"fmt"
 	"net/http"
+	"os"
+	"sync"
 
 	"github.com/docker/distribution/registry/auth/token"
 	httpauth "github.com/goharbor/harbor/src/common/http/modifier/auth"
@@ -10,6 +12,9 @@ import (
 	"github.com/goharbor/harbor/src/common/utils/registry/auth"
 )
 
+var coreClient *http.Client
+var mutex = &sync.Mutex{}
+
 // NewRepositoryClient creates a repository client with standard token authorizer
 func NewRepositoryClient(endpoint string, insecure bool, credential auth.Credential,
 	tokenServiceEndpoint, repository string) (*registry.Repository, error) {
@@ -79,3 +84,20 @@ func GetTokenForRepo(repository, secret, internalTokenServiceURL string) (string
 
 	return t.Token, nil
 }
+
+// GetClient returns the HTTP client that will attach jobservce secret to the request, which can be used for
+// accessing Harbor's Core Service.
+// This function returns error if the secret of Job service is not set.
+func GetClient() (*http.Client, error) {
+	mutex.Lock()
+	defer mutex.Unlock()
+	if coreClient == nil {
+		secret := os.Getenv("JOBSERVICE_SECRET")
+		if len(secret) == 0 {
+			return nil, fmt.Errorf("unable to load secret for job service")
+		}
+		modifier := httpauth.NewSecretAuthorizer(secret)
+		coreClient = &http.Client{Transport: registry.NewTransport(&http.Transport{}, modifier)}
+	}
+	return coreClient, nil
+}

src/jobservice/job/impl/utils/utils_test.go

@@ -0,0 +1,45 @@
+// Copyright 2018 The Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package utils
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/goharbor/harbor/src/common/secret"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetClient(t *testing.T) {
+	assert := assert.New(t)
+	os.Setenv("", "")
+	_, err := GetClient()
+	assert.NotNil(err, "Error should be thrown if secret is not set")
+	os.Setenv("JOBSERVICE_SECRET", "thesecret")
+	c, err := GetClient()
+	assert.Nil(err)
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		v := r.Header.Get("Authorization")
+		assert.Equal(secret.HeaderPrefix+"thesecret", v)
+	}))
+	defer ts.Close()
+	c.Get(ts.URL)
+
+	os.Setenv("", "")
+	_, err = GetClient()
+	assert.Nil(err, "Error should be nil once client is initialized")
+
+}

src/jobservice/runtime/bootstrap.go

@@ -184,11 +184,12 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool(ctx *env.Context, cfg *config.Con
 	}
 	if err := redisWorkerPool.RegisterJobs(
 		map[string]interface{}{
-			job.ImageScanJob:   (*scan.ClairJob)(nil),
+			job.ImageScanJob:    (*scan.ClairJob)(nil),
-			job.ImageTransfer:  (*replication.Transfer)(nil),
+			job.ImageScanAllJob: (*scan.All)(nil),
-			job.ImageDelete:    (*replication.Deleter)(nil),
+			job.ImageTransfer:   (*replication.Transfer)(nil),
-			job.ImageReplicate: (*replication.Replicator)(nil),
+			job.ImageDelete:     (*replication.Deleter)(nil),
-			job.ImageGC:        (*gc.GarbageCollector)(nil),
+			job.ImageReplicate:  (*replication.Replicator)(nil),
+			job.ImageGC:         (*gc.GarbageCollector)(nil),
 		}); err != nil {
 		// exit
 		return nil, err