handlers for image scan, store results overview in DB

make/common/db/registry.sql

@@ -180,6 +180,20 @@ create table img_scan_job (
  PRIMARY KEY (id)
  );
 
+create table img_scan_overview (
+ image_digest varchar(128) NOT NULL,
+ scan_job_id int NOT NULL,
+ /* 0 indicates none, the higher the number, the more severe the status */
+ severity int NOT NULL default 0,
+ /* the json string to store components severity status, currently use a json to be more flexible and avoid creating additional tables. */
+ components_overview varchar(2048),
+ /* primary key for querying details, in clair it should be the name of the "top layer" */
+ details_key varchar(128),
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
+ PRIMARY KEY(image_digest)
+ );
+
 create table properties (
  k varchar(64) NOT NULL,
  v varchar(128) NOT NULL,

make/common/db/registry_sqlite.sql

@@ -171,6 +171,19 @@ create table img_scan_job (
  update_time timestamp default CURRENT_TIMESTAMP
  );
 
+create table img_scan_overview (
+ image_digest varchar(128) PRIMARY KEY,
+ scan_job_id int NOT NULL,
+ /* 0 indicates none, the higher the number, the more severe the status */
+ severity int NOT NULL default 0,
+ /* the json string to store components severity status, currently use a json to be more flexible and avoid creating additional tables. */
+ components_overview varchar(2048),
+ /* primary key for querying details, in clair it should be the name of the "top layer" */
+ details_key varchar(128),
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ update_time timestamp default CURRENT_TIMESTAMP 
+ );
+
 CREATE INDEX policy ON replication_job (policy_id);
 CREATE INDEX poid_uptime ON replication_job (policy_id, update_time);
  

make/common/templates/clair/config.yaml

@@ -16,7 +16,7 @@ clair:
     # Deadline before an API request will respond with a 503
     timeout: 300s
   updater:
-    interval: 0h
+    interval: 2h
 
   notifier:
     attempts: 3

make/docker-compose.clair.yml

@@ -8,6 +8,9 @@ services:
   jobservice:
     networks:
       - harbor-clair
+  registry:
+    networks:
+      - harbor-clair
   postgres:
     networks:
       harbor-clair:

src/common/dao/dao_test.go

@@ -134,6 +134,12 @@ const publicityOn = 1
 const publicityOff = 0
 
 func TestMain(m *testing.M) {
+	orm.Debug = true
+	f, err := os.Create("/root/jtdbtest.out")
+	if err != nil {
+		panic(err)
+	}
+	orm.DebugLog = orm.NewLog(f)
 	databases := []string{"mysql", "sqlite"}
 	for _, database := range databases {
 		log.Infof("run test cases for database: %s", database)
@@ -1693,3 +1699,40 @@ func TestUpdateScanJobStatus(t *testing.T) {
 	err = ClearTable(models.ScanJobTable)
 	assert.Nil(err)
 }
+
+func TestImgScanOverview(t *testing.T) {
+	assert := assert.New(t)
+	err := ClearTable(models.ScanOverviewTable)
+	assert.Nil(err)
+	digest := "sha256:0204dc6e09fa57ab99ac40e415eb637d62c8b2571ecbbc9ca0eb5e2ad2b5c56f"
+	res, err := GetImgScanOverview(digest)
+	assert.Nil(err)
+	assert.Nil(res)
+	err = SetScanJobForImg(digest, 33)
+	assert.Nil(err)
+	res, err = GetImgScanOverview(digest)
+	assert.Nil(err)
+	assert.Equal(int64(33), res.JobID)
+	err = SetScanJobForImg(digest, 22)
+	assert.Nil(err)
+	res, err = GetImgScanOverview(digest)
+	assert.Nil(err)
+	assert.Equal(int64(22), res.JobID)
+	pk := "22-sha256:sdfsdfarfwefwr23r43t34ggregergerger"
+	comp := &models.ComponentsOverview{
+		Total: 2,
+		Summary: []*models.ComponentsOverviewEntry{
+			&models.ComponentsOverviewEntry{
+				Sev:   int(models.SevMedium),
+				Count: 2,
+			},
+		},
+	}
+	err = UpdateImgScanOverview(digest, pk, models.SevMedium, comp)
+	assert.Nil(err)
+	res, err = GetImgScanOverview(digest)
+	assert.Nil(err)
+	assert.Equal(pk, res.DetailsKey)
+	assert.Equal(int(models.SevMedium), res.Sev)
+	assert.Equal(2, res.CompOverview.Summary[0].Count)
+}

src/common/dao/scan_job.go

@@ -18,6 +18,7 @@ import (
 	"github.com/astaxie/beego/orm"
 	"github.com/vmware/harbor/src/common/models"
 
+	"encoding/json"
 	"fmt"
 	"time"
 )
@@ -79,3 +80,69 @@ func scanJobQs(limit ...int) orm.QuerySeter {
 	}
 	return o.QueryTable(models.ScanJobTable).Limit(l)
 }
+
+// SetScanJobForImg updates the scan_job_id based on the digest of image, if there's no data, it created one record.
+func SetScanJobForImg(digest string, jobID int64) error {
+	o := GetOrmer()
+	rec := &models.ImgScanOverview{
+		Digest:     digest,
+		JobID:      jobID,
+		UpdateTime: time.Now(),
+	}
+	created, _, err := o.ReadOrCreate(rec, "Digest")
+	if err != nil {
+		return err
+	}
+	if !created {
+		rec.JobID = jobID
+		n, err := o.Update(rec, "JobID", "UpdateTime")
+		if n == 0 {
+			return fmt.Errorf("Failed to set scan job for image with digest: %s, error: %v", digest, err)
+		}
+	}
+	return nil
+}
+
+// GetImgScanOverview returns the ImgScanOverview based on the digest.
+func GetImgScanOverview(digest string) (*models.ImgScanOverview, error) {
+	o := GetOrmer()
+	rec := &models.ImgScanOverview{
+		Digest: digest,
+	}
+	err := o.Read(rec)
+	if err != nil && err != orm.ErrNoRows {
+		return nil, err
+	}
+	if err == orm.ErrNoRows {
+		return nil, nil
+	}
+	if len(rec.CompOverviewStr) > 0 {
+		co := &models.ComponentsOverview{}
+		if err := json.Unmarshal([]byte(rec.CompOverviewStr), co); err != nil {
+			return nil, err
+		}
+		rec.CompOverview = co
+	}
+	return rec, nil
+}
+
+// UpdateImgScanOverview updates the serverity and components status of a record in img_scan_overview
+func UpdateImgScanOverview(digest, detailsKey string, sev models.Severity, compOverview *models.ComponentsOverview) error {
+	o := GetOrmer()
+	b, err := json.Marshal(compOverview)
+	if err != nil {
+		return err
+	}
+	rec := &models.ImgScanOverview{
+		Digest:          digest,
+		Sev:             int(sev),
+		CompOverviewStr: string(b),
+		DetailsKey:      detailsKey,
+		UpdateTime:      time.Now(),
+	}
+	n, err := o.Update(rec, "Sev", "CompOverviewStr", "DetailsKey", "UpdateTime")
+	if n == 0 || err != nil {
+		return fmt.Errorf("Failed to update scan overview record with digest: %s, error: %v", digest, err)
+	}
+	return nil
+}

src/common/models/base.go

@@ -27,5 +27,6 @@ func init() {
 		new(Role),
 		new(AccessLog),
 		new(ScanJob),
-		new(RepoRecord))
+		new(RepoRecord),
+		new(ImgScanOverview))
 }

src/common/models/scan_job.go

@@ -19,6 +19,9 @@ import "time"
 //ScanJobTable is the name of the table whose data is mapped by ScanJob struct.
 const ScanJobTable = "img_scan_job"
 
+//ScanOverviewTable is the name of the table whose data is mapped by ImgScanOverview struct.
+const ScanOverviewTable = "img_scan_overview"
+
 //ScanJob is the model to represent a job for image scan in DB.
 type ScanJob struct {
 	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
@@ -30,7 +33,50 @@ type ScanJob struct {
 	UpdateTime   time.Time `orm:"column(update_time);auto_now" json:"update_time"`
 }
 
+// Severity represents the severity of a image/component in terms of vulnerability.
+type Severity int64
+
+// Sevxxx is the list of severity of image after scanning.
+const (
+	_ Severity = iota
+	SevNone
+	SevUnknown
+	SevLow
+	SevMedium
+	SevHigh
+)
+
 //TableName is required by by beego orm to map ScanJob to table img_scan_job
 func (s *ScanJob) TableName() string {
 	return ScanJobTable
 }
+
+//ImgScanOverview mapped to a record of image scan overview.
+type ImgScanOverview struct {
+	Digest          string              `orm:"pk;column(image_digest)" json:"image_digest"`
+	Status          string              `orm:"-" json:"scan_status"`
+	JobID           int64               `orm:"column(scan_job_id)" json:"job_id"`
+	Sev             int                 `orm:"column(severity)" json:"severity"`
+	CompOverviewStr string              `orm:"column(components_overview)" json:"-"`
+	CompOverview    *ComponentsOverview `orm:"-" json:"components"`
+	DetailsKey      string              `orm:"column(details_key)" json:"details_key"`
+	CreationTime    time.Time           `orm:"column(creation_time);auto_now_add" json:"creation_time"`
+	UpdateTime      time.Time           `orm:"column(update_time);auto_now" json:"update_time"`
+}
+
+//TableName ...
+func (iso *ImgScanOverview) TableName() string {
+	return ScanOverviewTable
+}
+
+//ComponentsOverview has the total number and a list of components number of different serverity level.
+type ComponentsOverview struct {
+	Total   int                        `json:"total"`
+	Summary []*ComponentsOverviewEntry `json:"summary"`
+}
+
+//ComponentsOverviewEntry ...
+type ComponentsOverviewEntry struct {
+	Sev   int `json:"severity"`
+	Count int `json:"count"`
+}

src/common/models/token.go

@@ -0,0 +1,22 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+// Token represents the json returned by registry token service
+type Token struct {
+	Token     string `json:"token"`
+	ExpiresIn int    `json:"expires_in"`
+	IssuedAt  string `json:"issued_at"`
+}

src/common/utils/clair/client.go

@@ -0,0 +1,108 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clair
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	//	"path"
+
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils/log"
+)
+
+// Client communicates with clair endpoint to scan image and get detailed scan result
+type Client struct {
+	endpoint string
+	//need to customize the logger to write output to job log.
+	logger *log.Logger
+	client *http.Client
+}
+
+// NewClient creates a new instance of client, set the logger as the job's logger if it's used in a job handler.
+func NewClient(endpoint string, logger *log.Logger) *Client {
+	if logger == nil {
+		logger = log.DefaultLogger()
+	}
+	return &Client{
+		endpoint: endpoint,
+		logger:   logger,
+		client:   &http.Client{},
+	}
+}
+
+// ScanLayer calls Clair's API to scan a layer.
+func (c *Client) ScanLayer(l models.ClairLayer) error {
+	layer := models.ClairLayerEnvelope{
+		Layer: &l,
+		Error: nil,
+	}
+	data, err := json.Marshal(layer)
+	if err != nil {
+		return err
+	}
+	c.logger.Infof("endpoint: %s", c.endpoint)
+	c.logger.Infof("body: %s", string(data))
+	req, err := http.NewRequest("POST", c.endpoint+"/v1/layers", bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+	req.Header.Set(http.CanonicalHeaderKey("Content-Type"), "application/json")
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	c.logger.Infof("response code: %d", resp.StatusCode)
+	if resp.StatusCode != http.StatusCreated {
+		c.logger.Warningf("Unexpected status code: %d", resp.StatusCode)
+		b, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
+	}
+	c.logger.Infof("Returning.")
+	return nil
+}
+
+// GetResult calls Clair's API to get layers with detailed vulnerability list
+func (c *Client) GetResult(layerName string) (*models.ClairLayerEnvelope, error) {
+	req, err := http.NewRequest("GET", c.endpoint+"/v1/layers/"+layerName+"?features&vulnerabilities", nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
+	}
+	var res models.ClairLayerEnvelope
+	err = json.Unmarshal(b, &res)
+	if err != nil {
+		return nil, err
+	}
+	return &res, nil
+}

src/common/utils/clair/utils.go

@@ -0,0 +1,37 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clair
+
+import (
+	"github.com/vmware/harbor/src/common/models"
+	"strings"
+)
+
+// ParseClairSev parse the severity of clair to Harbor's Severity type if the string is not recognized the value will be set to unknown.
+func ParseClairSev(clairSev string) models.Severity {
+	sev := strings.ToLower(clairSev)
+	switch sev {
+	case "negligible":
+		return models.SevNone
+	case "low":
+		return models.SevLow
+	case "medium":
+		return models.SevMedium
+	case "high":
+		return models.SevHigh
+	default:
+		return models.SevUnknown
+	}
+}

src/common/utils/clair/utils_test.go

@@ -0,0 +1,35 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package clair
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/vmware/harbor/src/common/models"
+)
+
+func TestParseServerity(t *testing.T) {
+	assert := assert.New(t)
+	in := map[string]models.Severity{
+		"negligible": models.SevNone,
+		"whatever":   models.SevUnknown,
+		"LOW":        models.SevLow,
+		"Medium":     models.SevMedium,
+		"high":       models.SevHigh,
+	}
+	for k, v := range in {
+		assert.Equal(v, ParseClairSev(k))
+	}
+}

src/common/utils/log/logger.go

@@ -64,6 +64,11 @@ func New(out io.Writer, fmtter Formatter, lvl Level) *Logger {
 	}
 }
 
+//DefaultLogger returns the default logger within the pkg, i.e. the one used in log.Infof....
+func DefaultLogger() *Logger {
+	return logger
+}
+
 //SetOutput sets the output of Logger l
 func (l *Logger) SetOutput(out io.Writer) {
 	l.mu.Lock()

src/common/utils/registry/auth/tokenauthorizer.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	//"github.com/vmware/harbor/src/common/config"
+	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/common/utils/registry"
 	registry_error "github.com/vmware/harbor/src/common/utils/error"
@@ -205,11 +206,7 @@ func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes []
 		return
 	}
 
-	tk := struct {
-		Token     string `json:"token"`
-		ExpiresIn int    `json:"expires_in"`
-		IssuedAt  string `json:"issued_at"`
-	}{}
+	tk := models.Token{}
 	if err = json.Unmarshal(b, &tk); err != nil {
 		return
 	}

src/jobservice/config/config.go

@@ -167,3 +167,8 @@ func ExtEndpoint() (string, error) {
 func InternalTokenServiceEndpoint() string {
 	return "http://ui/service/token"
 }
+
+// ClairEndpoint returns the end point of clair instance, by default it's the one deployed within Harbor.
+func ClairEndpoint() string {
+	return "http://clair:6060"
+}

src/jobservice/job/job_test.go

@@ -217,5 +217,11 @@ func prepareScanJobData() error {
 }
 
 func clearScanJobData() error {
-	return dao.ClearTable(models.ScanJobTable)
+	if err := dao.ClearTable(models.ScanJobTable); err != nil {
+		return err
+	}
+	if err := dao.ClearTable(models.ScanOverviewTable); err != nil {
+		return err
+	}
+	return nil
 }

src/jobservice/job/jobs.go

@@ -220,6 +220,10 @@ func (sj *ScanJob) Init() error {
 		Tag:        job.Tag,
 		Digest:     job.Digest,
 	}
+	err = dao.SetScanJobForImg(job.Digest, sj.id)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 

src/jobservice/job/statemachine.go

@@ -258,11 +258,14 @@ func addImgScanTransition(sm *SM, parm *ScanJobParm) {
 		Repository: parm.Repository,
 		Tag:        parm.Tag,
 		Digest:     parm.Digest,
+		JobID:      sm.CurrentJob.ID(),
 		Logger:     sm.Logger,
 	}
 
+	layerScanHandler := &scan.LayerScanHandler{Context: ctx}
 	sm.AddTransition(models.JobRunning, scan.StateInitialize, &scan.Initializer{Context: ctx})
-	sm.AddTransition(scan.StateInitialize, scan.StateScanLayer, &scan.LayerScanHandler{Context: ctx})
+	sm.AddTransition(scan.StateInitialize, scan.StateScanLayer, layerScanHandler)
+	sm.AddTransition(scan.StateScanLayer, scan.StateScanLayer, layerScanHandler)
 	sm.AddTransition(scan.StateScanLayer, scan.StateSummarize, &scan.SummarizeHandler{Context: ctx})
 	sm.AddTransition(scan.StateSummarize, models.JobFinished, &StatusUpdater{sm.CurrentJob, models.JobFinished})
 }

src/jobservice/scan/context.go

@@ -15,6 +15,8 @@
 package scan
 
 import (
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils/clair"
 	"github.com/vmware/harbor/src/common/utils/log"
 )
 
@@ -29,16 +31,15 @@ const (
 
 //JobContext is for sharing data across handlers in a execution of a scan job.
 type JobContext struct {
+	JobID      int64
 	Repository string
 	Tag        string
 	Digest     string
-	//the digests of layers
-	layers []string
-	//each layer name has to be unique, so it should be ${img-digest}-${layer-digest}
-	layerNames []string
-	//the index of current layer
+	//The array of data object to set as request body for layer scan.
+	layers  []models.ClairLayer
 	current int
 	//token for accessing the registry
 	token       string
+	clairClient *clair.Client
 	Logger      *log.Logger
 }

src/jobservice/scan/handlers.go

@@ -15,7 +15,17 @@
 package scan
 
 import (
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils/clair"
+	"github.com/vmware/harbor/src/common/utils/registry/auth"
+	"github.com/vmware/harbor/src/jobservice/config"
+	"github.com/vmware/harbor/src/jobservice/utils"
+
+	"fmt"
+	"net/http"
 )
 
 // Initializer will handle the initialise state pull the manifest, prepare token.
@@ -27,9 +37,60 @@ type Initializer struct {
 func (iz *Initializer) Enter() (string, error) {
 	logger := iz.Context.Logger
 	logger.Infof("Entered scan initializer")
+	regURL, err := config.LocalRegURL()
+	if err != nil {
+		logger.Errorf("Failed to read regURL, error: %v", err)
+		return "", err
+	}
+	c := &http.Cookie{Name: models.UISecretCookie, Value: config.JobserviceSecret()}
+	repoClient, err := utils.NewRepositoryClient(regURL, false, auth.NewCookieCredential(c),
+		config.InternalTokenServiceEndpoint(), iz.Context.Repository, "pull")
+	if err != nil {
+		logger.Errorf("An error occurred while creating repository client: %v", err)
+		return "", err
+	}
+
+	_, _, payload, err := repoClient.PullManifest(iz.Context.Digest, []string{schema2.MediaTypeManifest})
+	if err != nil {
+		logger.Errorf("Error pulling manifest for image %s:%s :%v", iz.Context.Repository, iz.Context.Tag, err)
+		return "", err
+	}
+	manifest, _, err := distribution.UnmarshalManifest(schema2.MediaTypeManifest, payload)
+	if err != nil {
+		logger.Error("Failed to unMarshal manifest from response")
+		return "", err
+	}
+
+	tk, err := utils.GetTokenForRepo(iz.Context.Repository)
+	if err != nil {
+		return "", err
+	}
+	iz.Context.token = tk
+	iz.Context.clairClient = clair.NewClient(config.ClairEndpoint(), logger)
+	iz.prepareLayers(regURL, manifest.References())
 	return StateScanLayer, nil
 }
 
+func (iz *Initializer) prepareLayers(registryEndpoint string, descriptors []distribution.Descriptor) {
+	//	logger := iz.Context.Logger
+	tokenHeader := map[string]string{"Authorization": fmt.Sprintf("Bearer %s", iz.Context.token)}
+	for _, d := range descriptors {
+		if d.MediaType == schema2.MediaTypeConfig {
+			continue
+		}
+		l := models.ClairLayer{
+			Name:    fmt.Sprintf("%d-%s", iz.Context.JobID, d.Digest),
+			Headers: tokenHeader,
+			Format:  "Docker",
+			Path:    utils.BuildBlobURL(registryEndpoint, iz.Context.Repository, string(d.Digest)),
+		}
+		if len(iz.Context.layers) > 0 {
+			l.ParentName = iz.Context.layers[len(iz.Context.layers)-1].Name
+		}
+		iz.Context.layers = append(iz.Context.layers, l)
+	}
+}
+
 // Exit ...
 func (iz *Initializer) Exit() error {
 	return nil
@@ -43,9 +104,20 @@ type LayerScanHandler struct {
 // Enter ...
 func (ls *LayerScanHandler) Enter() (string, error) {
 	logger := ls.Context.Logger
-	logger.Infof("Entered scan layer handler")
+	currentLayer := ls.Context.layers[ls.Context.current]
+	logger.Infof("Entered scan layer handler, current: %d, layer name: %s", ls.Context.current, currentLayer.Name)
+	err := ls.Context.clairClient.ScanLayer(currentLayer)
+	if err != nil {
+		logger.Errorf("Unexpected error: %v", err)
+		return "", err
+	}
+	ls.Context.current++
+	if ls.Context.current == len(ls.Context.layers) {
 		return StateSummarize, nil
 	}
+	logger.Infof("After scanning, return with next state: %s", StateScanLayer)
+	return StateScanLayer, nil
+}
 
 // Exit ...
 func (ls *LayerScanHandler) Exit() error {
@@ -61,6 +133,46 @@ type SummarizeHandler struct {
 func (sh *SummarizeHandler) Enter() (string, error) {
 	logger := sh.Context.Logger
 	logger.Infof("Entered summarize handler")
+	layerName := sh.Context.layers[len(sh.Context.layers)-1].Name
+	logger.Infof("Top layer's name: %s, will use it to get the vulnerability result of image", layerName)
+	res, err := sh.Context.clairClient.GetResult(layerName)
+	if err != nil {
+		logger.Errorf("Failed to get result from Clair, error: %v", err)
+		return "", err
+	}
+	vulnMap := make(map[models.Severity]int)
+	features := res.Layer.Features
+	totalComponents := len(features)
+	logger.Infof("total features: %d", totalComponents)
+	var temp models.Severity
+	for _, f := range features {
+		sev := models.SevNone
+		for _, v := range f.Vulnerabilities {
+			temp = clair.ParseClairSev(v.Severity)
+			if temp > sev {
+				sev = temp
+			}
+		}
+		logger.Infof("Feature: %s, Severity: %d", f.Name, sev)
+		vulnMap[sev]++
+	}
+	overallSev := models.SevNone
+	compSummary := []*models.ComponentsOverviewEntry{}
+	for k, v := range vulnMap {
+		if k > overallSev {
+			overallSev = k
+		}
+		entry := &models.ComponentsOverviewEntry{
+			Sev:   int(k),
+			Count: v,
+		}
+		compSummary = append(compSummary, entry)
+	}
+	compOverview := &models.ComponentsOverview{
+		Total:   totalComponents,
+		Summary: compSummary,
+	}
+	err = dao.UpdateImgScanOverview(sh.Context.Digest, layerName, overallSev, compOverview)
 	return models.JobFinished, nil
 }
 

src/jobservice/utils/utils.go

@@ -15,9 +15,16 @@
 package utils
 
 import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+
+	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/utils/registry"
 	"github.com/vmware/harbor/src/common/utils/registry/auth"
-	"net/http"
+	"github.com/vmware/harbor/src/jobservice/config"
 )
 
 //NewRepositoryClient create a repository client with scope type "reopsitory" and scope as the repository it would access.
@@ -51,3 +58,44 @@ func (u *userAgentModifier) Modify(req *http.Request) error {
 	req.Header.Set(http.CanonicalHeaderKey("User-Agent"), u.userAgent)
 	return nil
 }
+
+// BuildBlobURL ...
+func BuildBlobURL(endpoint, repository, digest string) string {
+	return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repository, digest)
+}
+
+//GetTokenForRepo is a temp solution for job handler to get a token for clair.
+func GetTokenForRepo(repository string) (string, error) {
+	u, err := url.Parse(config.InternalTokenServiceEndpoint())
+	if err != nil {
+		return "", err
+	}
+	q := u.Query()
+	q.Add("service", "harbor-registry")
+	q.Add("scope", fmt.Sprintf("repository:%s:pull", repository))
+	u.RawQuery = q.Encode()
+	r, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return "", err
+	}
+	c := &http.Cookie{Name: models.UISecretCookie, Value: config.JobserviceSecret()}
+	r.AddCookie(c)
+	client := &http.Client{}
+	resp, err := client.Do(r)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("Unexpected response from token service, code: %d, %s", resp.StatusCode, string(b))
+	}
+	tk := models.Token{}
+	if err := json.Unmarshal(b, &tk); err != nil {
+		return "", err
+	}
+	return tk.Token, nil
+}

src/ui/service/token/authutils.go

@@ -23,12 +23,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/docker/distribution/registry/auth/token"
+	"github.com/docker/libtrust"
+	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/security"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/ui/config"
-
-	"github.com/docker/distribution/registry/auth/token"
-	"github.com/docker/libtrust"
 )
 
 const (
@@ -132,18 +132,16 @@ func MakeRawToken(username, service string, access []*token.ResourceActions) (to
 	return rs, expiresIn, issuedAt, nil
 }
 
-type tokenJSON struct {
-	Token     string `json:"token"`
-	ExpiresIn int    `json:"expires_in"`
-	IssuedAt  string `json:"issued_at"`
-}
-
-func makeToken(username, service string, access []*token.ResourceActions) (*tokenJSON, error) {
+func makeToken(username, service string, access []*token.ResourceActions) (*models.Token, error) {
 	raw, expires, issued, err := MakeRawToken(username, service, access)
 	if err != nil {
 		return nil, err
 	}
-	return &tokenJSON{raw, expires, issued.Format(time.RFC3339)}, nil
+	return &models.Token{
+		Token:     raw,
+		ExpiresIn: expires,
+		IssuedAt:  issued.Format(time.RFC3339),
+	}, nil
 }
 
 func permToActions(p string) []string {

src/ui/service/token/creator.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 
 	"github.com/docker/distribution/registry/auth/token"
+	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/security"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/ui/config"
@@ -70,7 +71,7 @@ func InitCreators() {
 
 // Creator creates a token ready to be served based on the http request.
 type Creator interface {
-	Create(r *http.Request) (*tokenJSON, error)
+	Create(r *http.Request) (*models.Token, error)
 }
 
 type imageParser interface {
@@ -178,7 +179,7 @@ func (e *unauthorizedError) Error() string {
 	return "Unauthorized"
 }
 
-func (g generalCreator) Create(r *http.Request) (*tokenJSON, error) {
+func (g generalCreator) Create(r *http.Request) (*models.Token, error) {
 	var err error
 	scopes := parseScopes(r.URL)
 	log.Debugf("scopes: %v", scopes)

src/ui/service/token/token_test.go

@@ -171,7 +171,7 @@ func TestBasicParser(t *testing.T) {
 	for _, rec := range testList {
 		r, err := p.parse(rec.input)
 		if rec.expectError {
-			assert.Error(t, err, "Expected error for input: %s", rec.input)
+			assert.Error(t, err, fmt.Sprintf("Expected error for input: %s", rec.input))
 		} else {
 			assert.Nil(t, err, "Expected no error for input: %s", rec.input)
 			assert.Equal(t, rec.expect, *r, "result mismatch for input: %s", rec.input)
@@ -193,7 +193,7 @@ func TestEndpointParser(t *testing.T) {
 	for _, rec := range testList {
 		r, err := p.parse(rec.input)
 		if rec.expectError {
-			assert.Error(t, err, "Expected error for input: %s", rec.input)
+			assert.Error(t, err, fmt.Sprintf("Expected error for input: %s", rec.input))
 		} else {
 			assert.Nil(t, err, "Expected no error for input: %s", rec.input)
 			assert.Equal(t, rec.expect, *r, "result mismatch for input: %s", rec.input)

src/vendor/github.com/astaxie/beego/orm/orm.go

@@ -137,10 +137,11 @@ func (o *orm) ReadOrCreate(md interface{}, col1 string, cols ...string) (bool, i
 	if err == ErrNoRows {
 		// Create
 		id, err := o.Insert(md)
+		fmt.Printf("id when create: %d", id)
 		return (err == nil), id, err
 	}
 
-	return false, ind.FieldByIndex(mi.fields.pk.fieldIndex).Int(), err
+	return false, 0, err
 }
 
 // insert model data to database