diff --git a/src/common/utils/clair/utils.go b/src/common/utils/clair/utils.go index 4e1690e2f..d5ef006bb 100644 --- a/src/common/utils/clair/utils.go +++ b/src/common/utils/clair/utils.go @@ -95,3 +95,8 @@ func transformVuln(clairVuln *models.ClairLayerEnvelope) (*models.ComponentsOver Summary: compSummary, }, overallSev } + +//TransformVuln is for running scanning job in both job service V1 and V2. +func TransformVuln(clairVuln *models.ClairLayerEnvelope) (*models.ComponentsOverview, models.Severity) { + return transformVuln(clairVuln) +} diff --git a/src/jobservice_v2/job/impl/scan/.job.go.swp b/src/jobservice_v2/job/impl/scan/.job.go.swp new file mode 100644 index 000000000..0551120f9 Binary files /dev/null and b/src/jobservice_v2/job/impl/scan/.job.go.swp differ diff --git a/src/jobservice_v2/job/impl/scan/job.go b/src/jobservice_v2/job/impl/scan/job.go new file mode 100644 index 000000000..15a00842f --- /dev/null +++ b/src/jobservice_v2/job/impl/scan/job.go @@ -0,0 +1,139 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scan + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + + "github.com/docker/distribution" + "github.com/docker/distribution/manifest/schema2" + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/job" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/clair" + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice_v2/env" + "github.com/vmware/harbor/src/jobservice_v2/job/impl/utils" +) + +// ClairJob is the struct to scan Harbor's Image with Clair +type ClairJob struct { +} + +// MaxFails implements the interface in job/Interface +func (cj *ClairJob) MaxFails() uint { + return 1 +} + +// ShouldRetry implements the interface in job/Interface +func (cj *ClairJob) ShouldRetry() bool { + return false +} + +// Validate implements the interface in job/Interface +func (cj *ClairJob) Validate(params map[string]interface{}) error { + return nil +} + +// Run implements the interface in job/Interface +func (cj *ClairJob) Run(ctx env.JobContext, params map[string]interface{}) error { + // TODO: get logger from ctx? + logger := log.DefaultLogger() + + jobParms, err := transformParam(params) + if err != nil { + logger.Errorf("Failed to prepare parms for scan job, error: %v", err) + return err + } + + repoClient, err := utils.NewRepositoryClientForJobservice(jobParms.Repository, jobParms.RegistryURL, jobParms.Secret, jobParms.TokenEndpoint) + if err != nil { + return err + } + imgDigest, _, payload, err := repoClient.PullManifest(jobParms.Tag, []string{schema2.MediaTypeManifest}) + if err != nil { + logger.Errorf("Error pulling manifest for image %s:%s :%v", jobParms.Repository, jobParms.Tag, err) + return err + } + token, err := utils.GetTokenForRepo(jobParms.Repository, jobParms.Secret, jobParms.TokenEndpoint) + if err != nil { + logger.Errorf("Failed to get token, error: %v", err) + return err + } + layers, err := prepareLayers(payload, jobParms.RegistryURL, jobParms.Repository, token) + if err != nil { + logger.Errorf("Failed to prepare layers, error: %v", err) + return err + } + clairClient := clair.NewClient(jobParms.ClairEndpoint, logger) + + for _, l := range layers { + logger.Infof("Scanning Layer: %s, path: %s", l.Name, l.Path) + if err := clairClient.ScanLayer(l); err != nil { + logger.Errorf("Failed to scan layer: %s, error: %v", l.Name, err) + return err + } + } + + layerName := layers[len(layers)-1].Name + res, err := clairClient.GetResult(layerName) + if err != nil { + logger.Errorf("Failed to get result from Clair, error: %v", err) + return err + } + compOverview, sev := clair.TransformVuln(res) + err = dao.UpdateImgScanOverview(imgDigest, layerName, sev, compOverview) + return err +} + +func transformParam(params map[string]interface{}) (*job.ScanJobParms, error) { + res := job.ScanJobParms{} + parmsBytes, err := json.Marshal(params) + if err != nil { + return nil, err + } + err = json.Unmarshal(parmsBytes, &res) + return &res, err +} + +func prepareLayers(payload []byte, registryURL, repo, tk string) ([]models.ClairLayer, error) { + layers := []models.ClairLayer{} + manifest, _, err := distribution.UnmarshalManifest(schema2.MediaTypeManifest, payload) + if err != nil { + return layers, err + } + tokenHeader := map[string]string{"Connection": "close", "Authorization": fmt.Sprintf("Bearer %s", tk)} + // form the chain by using the digests of all parent layers in the image, such that if another image is built on top of this image the layer name can be re-used. + shaChain := "" + for _, d := range manifest.References() { + if d.MediaType == schema2.MediaTypeConfig { + continue + } + shaChain += string(d.Digest) + "-" + l := models.ClairLayer{ + Name: fmt.Sprintf("%x", sha256.Sum256([]byte(shaChain))), + Headers: tokenHeader, + Format: "Docker", + Path: utils.BuildBlobURL(registryURL, repo, string(d.Digest)), + } + if len(layers) > 0 { + l.ParentName = layers[len(layers)-1].Name + } + layers = append(layers, l) + } + return layers, nil +} diff --git a/src/jobservice_v2/job/impl/utils/utils.go b/src/jobservice_v2/job/impl/utils/utils.go new file mode 100644 index 000000000..758b03399 --- /dev/null +++ b/src/jobservice_v2/job/impl/utils/utils.go @@ -0,0 +1,85 @@ +package utils + +import ( + "fmt" + "net/http" + + "github.com/docker/distribution/registry/auth/token" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/registry" + "github.com/vmware/harbor/src/common/utils/registry/auth" +) + +// NewRepositoryClient creates a repository client with standard token authorizer +func NewRepositoryClient(endpoint string, insecure bool, credential auth.Credential, + tokenServiceEndpoint, repository string) (*registry.Repository, error) { + + transport := registry.GetHTTPTransport(insecure) + + authorizer := auth.NewStandardTokenAuthorizer(&http.Client{ + Transport: transport, + }, credential, tokenServiceEndpoint) + + uam := &userAgentModifier{ + userAgent: "harbor-registry-client", + } + + return registry.NewRepository(repository, endpoint, &http.Client{ + Transport: registry.NewTransport(transport, authorizer, uam), + }) +} + +// NewRepositoryClientForJobservice creates a repository client that can only be used to +// access the internal registry +func NewRepositoryClientForJobservice(repository, internalRegistryURL, secret, internalTokenServiceURL string) (*registry.Repository, error) { + transport := registry.GetHTTPTransport() + + credential := auth.NewCookieCredential(&http.Cookie{ + Name: models.UISecretCookie, + Value: secret, + }) + + authorizer := auth.NewStandardTokenAuthorizer(&http.Client{ + Transport: transport, + }, credential, internalTokenServiceURL) + + uam := &userAgentModifier{ + userAgent: "harbor-registry-client", + } + + return registry.NewRepository(repository, internalRegistryURL, &http.Client{ + Transport: registry.NewTransport(transport, authorizer, uam), + }) +} + +type userAgentModifier struct { + userAgent string +} + +// Modify adds user-agent header to the request +func (u *userAgentModifier) Modify(req *http.Request) error { + req.Header.Set(http.CanonicalHeaderKey("User-Agent"), u.userAgent) + return nil +} + +// BuildBlobURL ... +func BuildBlobURL(endpoint, repository, digest string) string { + return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repository, digest) +} + +//GetTokenForRepo is used for job handler to get a token for clair. +func GetTokenForRepo(repository, secret, internalTokenServiceURL string) (string, error) { + c := &http.Cookie{Name: models.UISecretCookie, Value: secret} + credentail := auth.NewCookieCredential(c) + t, err := auth.GetToken(internalTokenServiceURL, true, credentail, + []*token.ResourceActions{&token.ResourceActions{ + Type: "repository", + Name: repository, + Actions: []string{"pull"}, + }}) + if err != nil { + return "", err + } + + return t.Token, nil +} diff --git a/src/jobservice_v2/runtime/bootstrap.go b/src/jobservice_v2/runtime/bootstrap.go index 7fd8c04b3..edeb05985 100644 --- a/src/jobservice_v2/runtime/bootstrap.go +++ b/src/jobservice_v2/runtime/bootstrap.go @@ -10,12 +10,14 @@ import ( "syscall" "time" + "github.com/vmware/harbor/src/common/job" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/jobservice_v2/api" "github.com/vmware/harbor/src/jobservice_v2/config" "github.com/vmware/harbor/src/jobservice_v2/core" "github.com/vmware/harbor/src/jobservice_v2/env" "github.com/vmware/harbor/src/jobservice_v2/job/impl" + "github.com/vmware/harbor/src/jobservice_v2/job/impl/scan" "github.com/vmware/harbor/src/jobservice_v2/pool" ) @@ -141,6 +143,11 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool(ctx *env.Context, cfg config.Conf ctx.ErrorChan <- err return redisWorkerPool //avoid nil pointer issue } + if err := redisWorkerPool.RegisterJob(job.ImageScanJob, (*scan.ClairJob)(nil)); err != nil { + //exit + ctx.ErrorChan <- err + return redisWorkerPool //avoid nil pointer issue + } redisWorkerPool.Start()