Migrate scan job to job service V1 phase1

src/common/utils/clair/utils.go

@@ -95,3 +95,8 @@ func transformVuln(clairVuln *models.ClairLayerEnvelope) (*models.ComponentsOver
 		Summary: compSummary,
 	}, overallSev
 }
+
+//TransformVuln is for running scanning job in both job service V1 and V2.
+func TransformVuln(clairVuln *models.ClairLayerEnvelope) (*models.ComponentsOverview, models.Severity) {
+	return transformVuln(clairVuln)
+}

src/jobservice_v2/job/impl/scan/job.go

@@ -0,0 +1,139 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scan
+
+import (
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/vmware/harbor/src/common/dao"
+	"github.com/vmware/harbor/src/common/job"
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils/clair"
+	"github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/jobservice_v2/env"
+	"github.com/vmware/harbor/src/jobservice_v2/job/impl/utils"
+)
+
+// ClairJob is the struct to scan Harbor's Image with Clair
+type ClairJob struct {
+}
+
+// MaxFails implements the interface in job/Interface
+func (cj *ClairJob) MaxFails() uint {
+	return 1
+}
+
+// ShouldRetry implements the interface in job/Interface
+func (cj *ClairJob) ShouldRetry() bool {
+	return false
+}
+
+// Validate implements the interface in job/Interface
+func (cj *ClairJob) Validate(params map[string]interface{}) error {
+	return nil
+}
+
+// Run implements the interface in job/Interface
+func (cj *ClairJob) Run(ctx env.JobContext, params map[string]interface{}) error {
+	// TODO: get logger from ctx?
+	logger := log.DefaultLogger()
+
+	jobParms, err := transformParam(params)
+	if err != nil {
+		logger.Errorf("Failed to prepare parms for scan job, error: %v", err)
+		return err
+	}
+
+	repoClient, err := utils.NewRepositoryClientForJobservice(jobParms.Repository, jobParms.RegistryURL, jobParms.Secret, jobParms.TokenEndpoint)
+	if err != nil {
+		return err
+	}
+	imgDigest, _, payload, err := repoClient.PullManifest(jobParms.Tag, []string{schema2.MediaTypeManifest})
+	if err != nil {
+		logger.Errorf("Error pulling manifest for image %s:%s :%v", jobParms.Repository, jobParms.Tag, err)
+		return err
+	}
+	token, err := utils.GetTokenForRepo(jobParms.Repository, jobParms.Secret, jobParms.TokenEndpoint)
+	if err != nil {
+		logger.Errorf("Failed to get token, error: %v", err)
+		return err
+	}
+	layers, err := prepareLayers(payload, jobParms.RegistryURL, jobParms.Repository, token)
+	if err != nil {
+		logger.Errorf("Failed to prepare layers, error: %v", err)
+		return err
+	}
+	clairClient := clair.NewClient(jobParms.ClairEndpoint, logger)
+
+	for _, l := range layers {
+		logger.Infof("Scanning Layer: %s, path: %s", l.Name, l.Path)
+		if err := clairClient.ScanLayer(l); err != nil {
+			logger.Errorf("Failed to scan layer: %s, error: %v", l.Name, err)
+			return err
+		}
+	}
+
+	layerName := layers[len(layers)-1].Name
+	res, err := clairClient.GetResult(layerName)
+	if err != nil {
+		logger.Errorf("Failed to get result from Clair, error: %v", err)
+		return err
+	}
+	compOverview, sev := clair.TransformVuln(res)
+	err = dao.UpdateImgScanOverview(imgDigest, layerName, sev, compOverview)
+	return err
+}
+
+func transformParam(params map[string]interface{}) (*job.ScanJobParms, error) {
+	res := job.ScanJobParms{}
+	parmsBytes, err := json.Marshal(params)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(parmsBytes, &res)
+	return &res, err
+}
+
+func prepareLayers(payload []byte, registryURL, repo, tk string) ([]models.ClairLayer, error) {
+	layers := []models.ClairLayer{}
+	manifest, _, err := distribution.UnmarshalManifest(schema2.MediaTypeManifest, payload)
+	if err != nil {
+		return layers, err
+	}
+	tokenHeader := map[string]string{"Connection": "close", "Authorization": fmt.Sprintf("Bearer %s", tk)}
+	// form the chain by using the digests of all parent layers in the image, such that if another image is built on top of this image the layer name can be re-used.
+	shaChain := ""
+	for _, d := range manifest.References() {
+		if d.MediaType == schema2.MediaTypeConfig {
+			continue
+		}
+		shaChain += string(d.Digest) + "-"
+		l := models.ClairLayer{
+			Name:    fmt.Sprintf("%x", sha256.Sum256([]byte(shaChain))),
+			Headers: tokenHeader,
+			Format:  "Docker",
+			Path:    utils.BuildBlobURL(registryURL, repo, string(d.Digest)),
+		}
+		if len(layers) > 0 {
+			l.ParentName = layers[len(layers)-1].Name
+		}
+		layers = append(layers, l)
+	}
+	return layers, nil
+}

src/jobservice_v2/job/impl/utils/utils.go

@@ -0,0 +1,85 @@
+package utils
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/docker/distribution/registry/auth/token"
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils/registry"
+	"github.com/vmware/harbor/src/common/utils/registry/auth"
+)
+
+// NewRepositoryClient creates a repository client with standard token authorizer
+func NewRepositoryClient(endpoint string, insecure bool, credential auth.Credential,
+	tokenServiceEndpoint, repository string) (*registry.Repository, error) {
+
+	transport := registry.GetHTTPTransport(insecure)
+
+	authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
+		Transport: transport,
+	}, credential, tokenServiceEndpoint)
+
+	uam := &userAgentModifier{
+		userAgent: "harbor-registry-client",
+	}
+
+	return registry.NewRepository(repository, endpoint, &http.Client{
+		Transport: registry.NewTransport(transport, authorizer, uam),
+	})
+}
+
+// NewRepositoryClientForJobservice creates a repository client that can only be used to
+// access the internal registry
+func NewRepositoryClientForJobservice(repository, internalRegistryURL, secret, internalTokenServiceURL string) (*registry.Repository, error) {
+	transport := registry.GetHTTPTransport()
+
+	credential := auth.NewCookieCredential(&http.Cookie{
+		Name:  models.UISecretCookie,
+		Value: secret,
+	})
+
+	authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
+		Transport: transport,
+	}, credential, internalTokenServiceURL)
+
+	uam := &userAgentModifier{
+		userAgent: "harbor-registry-client",
+	}
+
+	return registry.NewRepository(repository, internalRegistryURL, &http.Client{
+		Transport: registry.NewTransport(transport, authorizer, uam),
+	})
+}
+
+type userAgentModifier struct {
+	userAgent string
+}
+
+// Modify adds user-agent header to the request
+func (u *userAgentModifier) Modify(req *http.Request) error {
+	req.Header.Set(http.CanonicalHeaderKey("User-Agent"), u.userAgent)
+	return nil
+}
+
+// BuildBlobURL ...
+func BuildBlobURL(endpoint, repository, digest string) string {
+	return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repository, digest)
+}
+
+//GetTokenForRepo is used for job handler to get a token for clair.
+func GetTokenForRepo(repository, secret, internalTokenServiceURL string) (string, error) {
+	c := &http.Cookie{Name: models.UISecretCookie, Value: secret}
+	credentail := auth.NewCookieCredential(c)
+	t, err := auth.GetToken(internalTokenServiceURL, true, credentail,
+		[]*token.ResourceActions{&token.ResourceActions{
+			Type:    "repository",
+			Name:    repository,
+			Actions: []string{"pull"},
+		}})
+	if err != nil {
+		return "", err
+	}
+
+	return t.Token, nil
+}

src/jobservice_v2/runtime/bootstrap.go

@@ -10,12 +10,14 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/vmware/harbor/src/common/job"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/jobservice_v2/api"
 	"github.com/vmware/harbor/src/jobservice_v2/config"
 	"github.com/vmware/harbor/src/jobservice_v2/core"
 	"github.com/vmware/harbor/src/jobservice_v2/env"
 	"github.com/vmware/harbor/src/jobservice_v2/job/impl"
+	"github.com/vmware/harbor/src/jobservice_v2/job/impl/scan"
 	"github.com/vmware/harbor/src/jobservice_v2/pool"
 )
 
@@ -141,6 +143,11 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool(ctx *env.Context, cfg config.Conf
 		ctx.ErrorChan <- err
 		return redisWorkerPool //avoid nil pointer issue
 	}
+	if err := redisWorkerPool.RegisterJob(job.ImageScanJob, (*scan.ClairJob)(nil)); err != nil {
+		//exit
+		ctx.ErrorChan <- err
+		return redisWorkerPool //avoid nil pointer issue
+	}
 
 	redisWorkerPool.Start()