Merge branch 'master' of https://github.com/goharbor/harbor into project-quota-dev

src/common/dao/scan_job.go

@@ -15,12 +15,11 @@
 package dao
 
 import (
+	"encoding/json"
+	"fmt"
 	"github.com/astaxie/beego/orm"
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/common/utils/log"
-
-	"encoding/json"
-	"fmt"
 	"time"
 )
 

src/common/models/cve_whitelist.go

@@ -33,6 +33,23 @@ type CVEWhitelistItem struct {
 }
 
 // TableName ...
-func (r *CVEWhitelist) TableName() string {
+func (c *CVEWhitelist) TableName() string {
 	return "cve_whitelist"
 }
+
+// CVESet returns the set of CVE id of the items in the whitelist to help filter the vulnerability list
+func (c *CVEWhitelist) CVESet() map[string]struct{} {
+	r := map[string]struct{}{}
+	for _, it := range c.Items {
+		r[it.CVEID] = struct{}{}
+	}
+	return r
+}
+
+// IsExpired returns whether the whitelist is expired
+func (c *CVEWhitelist) IsExpired() bool {
+	if c.ExpiresAt == nil {
+		return false
+	}
+	return time.Now().Unix() >= *c.ExpiresAt
+}

src/common/models/cve_whitelist_test.go

@@ -0,0 +1,72 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"github.com/stretchr/testify/assert"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestCVEWhitelist_All(t *testing.T) {
+	future := int64(4411494000)
+	now := time.Now().Unix()
+	cases := []struct {
+		input   CVEWhitelist
+		cveset  map[string]struct{}
+		expired bool
+	}{
+		{
+			input: CVEWhitelist{
+				ID:        1,
+				ProjectID: 0,
+				Items:     []CVEWhitelistItem{},
+			},
+			cveset:  map[string]struct{}{},
+			expired: false,
+		},
+		{
+			input: CVEWhitelist{
+				ID:        1,
+				ProjectID: 0,
+				Items:     []CVEWhitelistItem{},
+				ExpiresAt: &now,
+			},
+			cveset:  map[string]struct{}{},
+			expired: true,
+		},
+		{
+			input: CVEWhitelist{
+				ID:        2,
+				ProjectID: 3,
+				Items: []CVEWhitelistItem{
+					{CVEID: "CVE-1999-0067"},
+					{CVEID: "CVE-2016-7654321"},
+				},
+				ExpiresAt: &future,
+			},
+			cveset: map[string]struct{}{
+				"CVE-1999-0067":    {},
+				"CVE-2016-7654321": {},
+			},
+			expired: false,
+		},
+	}
+	for _, c := range cases {
+		assert.Equal(t, c.expired, c.input.IsExpired())
+		assert.True(t, reflect.DeepEqual(c.cveset, c.input.CVESet()))
+	}
+}

src/common/models/scan_job.go

@@ -34,31 +34,6 @@ type ScanJob struct {
 	UpdateTime   time.Time `orm:"column(update_time);auto_now" json:"update_time"`
 }
 
-// Severity represents the severity of a image/component in terms of vulnerability.
-type Severity int64
-
-// Sevxxx is the list of severity of image after scanning.
-const (
-	_ Severity = iota
-	SevNone
-	SevUnknown
-	SevLow
-	SevMedium
-	SevHigh
-)
-
-// String is the output function for sererity variable
-func (sev Severity) String() string {
-	name := []string{"negligible", "unknown", "low", "medium", "high"}
-	i := int64(sev)
-	switch {
-	case i >= 1 && i <= int64(SevHigh):
-		return name[i-1]
-	default:
-		return "unknown"
-	}
-}
-
 // TableName is required by by beego orm to map ScanJob to table img_scan_job
 func (s *ScanJob) TableName() string {
 	return ScanJobTable
@@ -101,17 +76,6 @@ type ImageScanReq struct {
 	Tag  string `json:"tag"`
 }
 
-// VulnerabilityItem is an item in the vulnerability result returned by vulnerability details API.
-type VulnerabilityItem struct {
-	ID          string   `json:"id"`
-	Severity    Severity `json:"severity"`
-	Pkg         string   `json:"package"`
-	Version     string   `json:"version"`
-	Description string   `json:"description"`
-	Link        string   `json:"link"`
-	Fixed       string   `json:"fixedVersion,omitempty"`
-}
-
 // ScanAllPolicy is represent the json request and object for scan all policy, the parm is het
 type ScanAllPolicy struct {
 	Type string                 `json:"type"`

src/common/models/sev.go

@@ -0,0 +1,26 @@
+package models
+
+// Severity represents the severity of a image/component in terms of vulnerability.
+type Severity int64
+
+// Sevxxx is the list of severity of image after scanning.
+const (
+	_ Severity = iota
+	SevNone
+	SevUnknown
+	SevLow
+	SevMedium
+	SevHigh
+)
+
+// String is the output function for severity variable
+func (sev Severity) String() string {
+	name := []string{"negligible", "unknown", "low", "medium", "high"}
+	i := int64(sev)
+	switch {
+	case i >= 1 && i <= int64(SevHigh):
+		return name[i-1]
+	default:
+		return "unknown"
+	}
+}

src/common/utils/oidc/helper.go

@@ -35,20 +35,14 @@ const googleEndpoint = "https://accounts.google.com"
 
 type providerHelper struct {
 	sync.Mutex
-	ep       endpoint
+	instance     atomic.Value
-	instance atomic.Value
+	setting      atomic.Value
-	setting  atomic.Value
+	creationTime time.Time
-}
-
-type endpoint struct {
-	url        string
-	VerifyCert bool
 }
 
 func (p *providerHelper) get() (*gooidc.Provider, error) {
 	if p.instance.Load() != nil {
-		s := p.setting.Load().(models.OIDCSetting)
+		if time.Now().Sub(p.creationTime) > 3*time.Second {
-		if s.Endpoint != p.ep.url || s.VerifyCert != p.ep.VerifyCert { // relevant settings have changed, need to re-create provider.
 			if err := p.create(); err != nil {
 				return nil, err
 			}
@@ -57,7 +51,7 @@ func (p *providerHelper) get() (*gooidc.Provider, error) {
 		p.Lock()
 		defer p.Unlock()
 		if p.instance.Load() == nil {
-			if err := p.reload(); err != nil {
+			if err := p.reloadSetting(); err != nil {
 				return nil, err
 			}
 			if err := p.create(); err != nil {
@@ -65,7 +59,7 @@ func (p *providerHelper) get() (*gooidc.Provider, error) {
 			}
 			go func() {
 				for {
-					if err := p.reload(); err != nil {
+					if err := p.reloadSetting(); err != nil {
 						log.Warningf("Failed to refresh configuration, error: %v", err)
 					}
 					time.Sleep(3 * time.Second)
@@ -73,10 +67,11 @@ func (p *providerHelper) get() (*gooidc.Provider, error) {
 			}()
 		}
 	}
+
 	return p.instance.Load().(*gooidc.Provider), nil
 }
 
-func (p *providerHelper) reload() error {
+func (p *providerHelper) reloadSetting() error {
 	conf, err := config.OIDCSetting()
 	if err != nil {
 		return fmt.Errorf("failed to load OIDC setting: %v", err)
@@ -96,10 +91,7 @@ func (p *providerHelper) create() error {
 		return fmt.Errorf("failed to create OIDC provider, error: %v", err)
 	}
 	p.instance.Store(provider)
-	p.ep = endpoint{
+	p.creationTime = time.Now()
-		url:        s.Endpoint,
-		VerifyCert: s.VerifyCert,
-	}
 	return nil
 }
 

src/common/utils/oidc/helper_test.go

@@ -49,21 +49,20 @@ func TestMain(m *testing.M) {
 func TestHelperLoadConf(t *testing.T) {
 	testP := &providerHelper{}
 	assert.Nil(t, testP.setting.Load())
-	err := testP.reload()
+	err := testP.reloadSetting()
 	assert.Nil(t, err)
 	assert.Equal(t, "test", testP.setting.Load().(models.OIDCSetting).Name)
-	assert.Equal(t, endpoint{}, testP.ep)
 }
 
 func TestHelperCreate(t *testing.T) {
 	testP := &providerHelper{}
-	err := testP.reload()
+	err := testP.reloadSetting()
 	assert.Nil(t, err)
 	assert.Nil(t, testP.instance.Load())
 	err = testP.create()
 	assert.Nil(t, err)
-	assert.EqualValues(t, "https://accounts.google.com", testP.ep.url)
 	assert.NotNil(t, testP.instance.Load())
+	assert.True(t, time.Now().Sub(testP.creationTime) < 2*time.Second)
 }
 
 func TestHelperGet(t *testing.T) {

src/core/api/repository.go

@@ -17,6 +17,7 @@ package api
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/goharbor/harbor/src/pkg/scan"
 	"io/ioutil"
 	"net/http"
 	"sort"
@@ -34,7 +35,6 @@ import (
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/common/rbac"
 	"github.com/goharbor/harbor/src/common/utils"
-	"github.com/goharbor/harbor/src/common/utils/clair"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/common/utils/notary"
 	"github.com/goharbor/harbor/src/common/utils/registry"
@@ -1036,21 +1036,9 @@ func (ra *RepositoryAPI) VulnerabilityDetails() {
 		ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
 		return
 	}
-	res := []*models.VulnerabilityItem{}
+	res, err := scan.VulnListByDigest(digest)
-	overview, err := dao.GetImgScanOverview(digest)
 	if err != nil {
-		ra.SendInternalServerError(fmt.Errorf("failed to get the scan overview, error: %v", err))
+		log.Errorf("Failed to get vulnerability list for image: %s:%s", repository, tag)
-		return
-	}
-	if overview != nil && len(overview.DetailsKey) > 0 {
-		clairClient := clair.NewClient(config.ClairEndpoint(), nil)
-		log.Debugf("The key for getting details: %s", overview.DetailsKey)
-		details, err := clairClient.GetResult(overview.DetailsKey)
-		if err != nil {
-			ra.SendInternalServerError(fmt.Errorf("Failed to get scan details from Clair, error: %v", err))
-			return
-		}
-		res = transformVulnerabilities(details)
 	}
 	ra.Data["json"] = res
 	ra.ServeJSON()

src/core/api/utils.go

@@ -24,7 +24,6 @@ import (
 	commonhttp "github.com/goharbor/harbor/src/common/http"
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/common/utils"
-	"github.com/goharbor/harbor/src/common/utils/clair"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/common/utils/registry"
 	"github.com/goharbor/harbor/src/common/utils/registry/auth"
@@ -279,35 +278,3 @@ func repositoryExist(name string, client *registry.Repository) (bool, error) {
 	}
 	return len(tags) != 0, nil
 }
-
-// transformVulnerabilities transforms the returned value of Clair API to a list of VulnerabilityItem
-func transformVulnerabilities(layerWithVuln *models.ClairLayerEnvelope) []*models.VulnerabilityItem {
-	res := []*models.VulnerabilityItem{}
-	l := layerWithVuln.Layer
-	if l == nil {
-		return res
-	}
-	features := l.Features
-	if features == nil {
-		return res
-	}
-	for _, f := range features {
-		vulnerabilities := f.Vulnerabilities
-		if vulnerabilities == nil {
-			continue
-		}
-		for _, v := range vulnerabilities {
-			vItem := &models.VulnerabilityItem{
-				ID:          v.Name,
-				Pkg:         f.Name,
-				Version:     f.Version,
-				Severity:    clair.ParseClairSev(v.Severity),
-				Fixed:       v.FixedBy,
-				Link:        v.Link,
-				Description: v.Description,
-			}
-			res = append(res, vItem)
-		}
-	}
-	return res
-}

src/core/filter/security.go

@@ -229,8 +229,10 @@ type oidcCliReqCtxModifier struct{}
 
 func (oc *oidcCliReqCtxModifier) Modify(ctx *beegoctx.Context) bool {
 	path := ctx.Request.URL.Path
-	if path != "/service/token" && !strings.HasPrefix(path, "/chartrepo/") {
+	if path != "/service/token" &&
-		log.Debug("OIDC CLI modifer only handles request by docker CLI or helm CLI")
+		!strings.HasPrefix(path, "/chartrepo/") &&
+		!strings.HasPrefix(path, "/api/chartrepo/") {
+		log.Debug("OIDC CLI modifier only handles request by docker CLI or helm CLI")
 		return false
 	}
 	if ctx.Request.Context().Value(AuthModeKey).(string) != common.OIDCAuth {

src/core/proxy/interceptors.go

@@ -2,7 +2,6 @@ package proxy
 
 import (
 	"encoding/json"
-
 	"github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/common/utils/clair"
@@ -11,6 +10,7 @@ import (
 	"github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/core/promgr"
 	coreutils "github.com/goharbor/harbor/src/core/utils"
+	"github.com/goharbor/harbor/src/pkg/scan"
 
 	"context"
 	"fmt"
@@ -303,27 +303,41 @@ func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request)
 		vh.next.ServeHTTP(rw, req)
 		return
 	}
-	overview, err := dao.GetImgScanOverview(img.digest)
+	// TODO: Get whitelist based on project setting
+	wl, err := dao.GetSysCVEWhitelist()
 	if err != nil {
-		log.Errorf("failed to get ImgScanOverview with repo: %s, reference: %s, digest: %s. Error: %v", img.repository, img.reference, img.digest, err)
+		log.Errorf("Failed to get the whitelist, error: %v", err)
-		http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get ImgScanOverview."), http.StatusPreconditionFailed)
+		http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get CVE whitelist."), http.StatusPreconditionFailed)
 		return
 	}
-	// severity is 0 means that the image fails to scan or not scanned successfully.
+	vl, err := scan.VulnListByDigest(img.digest)
-	if overview == nil || overview.Sev == 0 {
+	if err != nil {
-		log.Debugf("cannot get the image scan overview info, failing the response.")
+		log.Errorf("Failed to get the vulnerability list, error: %v", err)
-		http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Cannot get the image severity."), http.StatusPreconditionFailed)
+		http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get vulnerabilities."), http.StatusPreconditionFailed)
 		return
 	}
-	imageSev := overview.Sev
+	filtered := vl.ApplyWhitelist(*wl)
-	if imageSev >= int(projectVulnerableSeverity) {
+	msg := vh.filterMsg(img, filtered)
-		log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", models.Severity(imageSev), projectVulnerableSeverity)
+	log.Info(msg)
-		http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", models.Severity(imageSev), projectVulnerableSeverity)), http.StatusPreconditionFailed)
+	if int(vl.Severity()) >= int(projectVulnerableSeverity) {
+		log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", vl.Severity(), projectVulnerableSeverity)
+		http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", vl.Severity(), projectVulnerableSeverity)), http.StatusPreconditionFailed)
 		return
 	}
 	vh.next.ServeHTTP(rw, req)
 }
 
+func (vh vulnerableHandler) filterMsg(img imageInfo, filtered scan.VulnerabilityList) string {
+	filterMsg := fmt.Sprintf("Image: %s/%s:%s, digest: %s, vulnerabilities fitered by whitelist:", img.projectName, img.repository, img.reference, img.digest)
+	if len(filtered) == 0 {
+		filterMsg = fmt.Sprintf("%s none.", filterMsg)
+	}
+	for _, v := range filtered {
+		filterMsg = fmt.Sprintf("%s ID: %s, severity: %s;", filterMsg, v.ID, v.Severity)
+	}
+	return filterMsg
+}
+
 func matchNotaryDigest(img imageInfo) (bool, error) {
 	if NotaryEndpoint == "" {
 		NotaryEndpoint = config.InternalNotaryEndpoint()

src/core/proxy/proxy.go

@@ -38,7 +38,15 @@ func Init(urls ...string) error {
 		return err
 	}
 	Proxy = httputil.NewSingleHostReverseProxy(targetURL)
-	handlers = handlerChain{head: readonlyHandler{next: urlHandler{next: multipleManifestHandler{next: listReposHandler{next: contentTrustHandler{next: vulnerableHandler{next: Proxy}}}}}}}
+	handlers = handlerChain{
+		head: readonlyHandler{
+			next: urlHandler{
+				next: multipleManifestHandler{
+					next: listReposHandler{
+						next: contentTrustHandler{
+							next: vulnerableHandler{
+								next: Proxy,
+							}}}}}}}
 	return nil
 }
 

src/jobservice/common/rds/keys.go

@@ -34,7 +34,7 @@ func RedisKeyScheduled(namespace string) string {
 
 // RedisKeyLastPeriodicEnqueue returns key of timestamp if last periodic enqueue.
 func RedisKeyLastPeriodicEnqueue(namespace string) string {
-	return RedisNamespacePrefix(namespace) + "last_periodic_enqueue"
+	return RedisNamespacePrefix(namespace) + "last_periodic_enqueue_h"
 }
 
 // KeyNamespacePrefix returns the based key based on the namespace.

src/jobservice/period/enqueuer_test.go

@@ -16,6 +16,12 @@ package period
 import (
 	"context"
 	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+
 	"github.com/goharbor/harbor/src/jobservice/common/rds"
 	"github.com/goharbor/harbor/src/jobservice/common/utils"
 	"github.com/goharbor/harbor/src/jobservice/env"
@@ -26,9 +32,6 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
-	"sync"
-	"testing"
-	"time"
 )
 
 // EnqueuerTestSuite tests functions of enqueuer
@@ -89,19 +92,30 @@ func (suite *EnqueuerTestSuite) TestEnqueuer() {
 			suite.enqueuer.stopChan <- true
 		}()
 
-		<-time.After(1 * time.Second)
-
 		key := rds.RedisKeyScheduled(suite.namespace)
 		conn := suite.pool.Get()
 		defer func() {
 			_ = conn.Close()
 		}()
 
-		count, err := redis.Int(conn.Do("ZCARD", key))
+		tk := time.NewTicker(500 * time.Millisecond)
-		require.Nil(suite.T(), err, "count scheduled: nil error expected but got %s", err)
+		defer tk.Stop()
-		assert.Condition(suite.T(), func() bool {
+
-			return count > 0
+		for {
-		}, "count of scheduled jobs should be greater than 0 but got %d", count)
+			select {
+			case <-tk.C:
+				count, err := redis.Int(conn.Do("ZCARD", key))
+				require.Nil(suite.T(), err, "count scheduled: nil error expected but got %s", err)
+				if assert.Condition(suite.T(), func() (success bool) {
+					return count > 0
+				}, "at least one job should be scheduled for the periodic job policy") {
+					return
+				}
+			case <-time.After(15 * time.Second):
+				require.NoError(suite.T(), errors.New("timeout (15s): expect at 1 scheduled job but still get nothing"))
+				return
+			}
+		}
 	}()
 
 	err := suite.enqueuer.start()
@@ -112,7 +126,7 @@ func (suite *EnqueuerTestSuite) prepare() {
 	now := time.Now()
 	minute := now.Minute()
 
-	coreSpec := fmt.Sprintf("30,50 %d * * * *", minute+2)
+	coreSpec := fmt.Sprintf("0-59 %d * * * *", minute)
 
 	// Prepare one
 	p := &Policy{

src/pkg/scan/vuln.go

@@ -0,0 +1,136 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scan
+
+import (
+	"fmt"
+	"github.com/goharbor/harbor/src/common/dao"
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/common/utils/clair"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/core/config"
+	"reflect"
+)
+
+// VulnerabilityItem represents a vulnerability reported by scanner
+type VulnerabilityItem struct {
+	ID          string          `json:"id"`
+	Severity    models.Severity `json:"severity"`
+	Pkg         string          `json:"package"`
+	Version     string          `json:"version"`
+	Description string          `json:"description"`
+	Link        string          `json:"link"`
+	Fixed       string          `json:"fixedVersion,omitempty"`
+}
+
+// VulnerabilityList is a list of vulnerabilities, which should be scanner-agnostic
+type VulnerabilityList []VulnerabilityItem
+
+// ApplyWhitelist filters out the CVE defined in the whitelist in the parm.
+// It returns the items that are filtered for the caller to track or log.
+func (vl *VulnerabilityList) ApplyWhitelist(whitelist models.CVEWhitelist) VulnerabilityList {
+	filtered := VulnerabilityList{}
+	if whitelist.IsExpired() {
+		log.Info("The input whitelist is expired, skip filtering")
+		return filtered
+	}
+	s := whitelist.CVESet()
+	r := (*vl)[:0]
+	for _, v := range *vl {
+		if _, ok := s[v.ID]; ok {
+			log.Debugf("Filtered Vulnerability in whitelist, CVE ID: %s, severity: %s", v.ID, v.Severity)
+			filtered = append(filtered, v)
+		} else {
+			r = append(r, v)
+		}
+	}
+	val := reflect.ValueOf(vl)
+	val.Elem().SetLen(len(r))
+	return filtered
+}
+
+// Severity returns the highest severity of the vulnerabilities in the list
+func (vl *VulnerabilityList) Severity() models.Severity {
+	s := models.SevNone
+	for _, v := range *vl {
+		if v.Severity > s {
+			s = v.Severity
+		}
+	}
+	return s
+}
+
+// HasCVE returns whether the vulnerability list has the vulnerability with CVE ID in the parm
+func (vl *VulnerabilityList) HasCVE(id string) bool {
+	for _, v := range *vl {
+		if v.ID == id {
+			return true
+		}
+	}
+	return false
+}
+
+// VulnListFromClairResult transforms the returned value of Clair API to a VulnerabilityList
+func VulnListFromClairResult(layerWithVuln *models.ClairLayerEnvelope) VulnerabilityList {
+	res := VulnerabilityList{}
+	if layerWithVuln == nil {
+		return res
+	}
+	l := layerWithVuln.Layer
+	if l == nil {
+		return res
+	}
+	features := l.Features
+	if features == nil {
+		return res
+	}
+	for _, f := range features {
+		vulnerabilities := f.Vulnerabilities
+		if vulnerabilities == nil {
+			continue
+		}
+		for _, v := range vulnerabilities {
+			vItem := VulnerabilityItem{
+				ID:          v.Name,
+				Pkg:         f.Name,
+				Version:     f.Version,
+				Severity:    clair.ParseClairSev(v.Severity),
+				Fixed:       v.FixedBy,
+				Link:        v.Link,
+				Description: v.Description,
+			}
+			res = append(res, vItem)
+		}
+	}
+	return res
+}
+
+// VulnListByDigest returns the VulnerabilityList based on the scan result of artifact with the digest in the parm
+func VulnListByDigest(digest string) (VulnerabilityList, error) {
+	var res VulnerabilityList
+	overview, err := dao.GetImgScanOverview(digest)
+	if err != nil {
+		return res, err
+	}
+	if overview == nil || len(overview.DetailsKey) == 0 {
+		return res, fmt.Errorf("unable to get the scan result for digest: %s, the artifact is not scanned", digest)
+	}
+	c := clair.NewClient(config.ClairEndpoint(), nil)
+	clairRes, err := c.GetResult(overview.DetailsKey)
+	if err != nil {
+		return res, fmt.Errorf("failed to get scan result from Clair, error: %v", err)
+	}
+	return VulnListFromClairResult(clairRes), nil
+}

src/pkg/scan/vuln_test.go

@@ -0,0 +1,178 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scan
+
+import (
+	"github.com/goharbor/harbor/src/common/dao"
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+var (
+	past      = int64(1561967574)
+	vulnList1 = VulnerabilityList{}
+	vulnList2 = VulnerabilityList{
+		{ID: "CVE-2018-10754",
+			Severity: models.SevLow,
+			Pkg:      "ncurses",
+			Version:  "6.0+20161126-1+deb9u2",
+		},
+		{
+			ID:       "CVE-2018-6485",
+			Severity: models.SevHigh,
+			Pkg:      "glibc",
+			Version:  "2.24-11+deb9u4",
+		},
+	}
+	whiteList1 = models.CVEWhitelist{
+		ExpiresAt: &past,
+		Items: []models.CVEWhitelistItem{
+			{CVEID: "CVE-2018-6485"},
+		},
+	}
+	whiteList2 = models.CVEWhitelist{
+		Items: []models.CVEWhitelistItem{
+			{CVEID: "CVE-2018-6485"},
+		},
+	}
+	whiteList3 = models.CVEWhitelist{
+		Items: []models.CVEWhitelistItem{
+			{CVEID: "CVE-2018-6485"},
+			{CVEID: "CVE-2018-10754"},
+			{CVEID: "CVE-2019-12817"},
+		},
+	}
+)
+
+func TestMain(m *testing.M) {
+	dao.PrepareTestForPostgresSQL()
+	os.Exit(m.Run())
+}
+
+func TestVulnerabilityList_HasCVE(t *testing.T) {
+	cases := []struct {
+		input  VulnerabilityList
+		cve    string
+		result bool
+	}{
+		{
+			input:  vulnList1,
+			cve:    "CVE-2018-10754",
+			result: false,
+		},
+		{
+			input:  vulnList2,
+			cve:    "CVE-2018-10754",
+			result: true,
+		},
+	}
+	for _, c := range cases {
+		assert.Equal(t, c.result, c.input.HasCVE(c.cve))
+	}
+}
+
+func TestVulnerabilityList_Severity(t *testing.T) {
+	cases := []struct {
+		input  VulnerabilityList
+		expect models.Severity
+	}{
+		{
+			input:  vulnList1,
+			expect: models.SevNone,
+		}, {
+			input:  vulnList2,
+			expect: models.SevHigh,
+		},
+	}
+	for _, c := range cases {
+		assert.Equal(t, c.expect, c.input.Severity())
+	}
+}
+
+func TestVulnerabilityList_ApplyWhitelist(t *testing.T) {
+	cases := []struct {
+		vl             VulnerabilityList
+		wl             models.CVEWhitelist
+		expectFiltered VulnerabilityList
+		expectSev      models.Severity
+	}{
+		{
+			vl:             vulnList2,
+			wl:             whiteList1,
+			expectFiltered: VulnerabilityList{},
+			expectSev:      models.SevHigh,
+		},
+		{
+			vl: vulnList2,
+			wl: whiteList2,
+			expectFiltered: VulnerabilityList{
+				{
+					ID:       "CVE-2018-6485",
+					Severity: models.SevHigh,
+					Pkg:      "glibc",
+					Version:  "2.24-11+deb9u4",
+				},
+			},
+			expectSev: models.SevLow,
+		},
+		{
+			vl:             vulnList1,
+			wl:             whiteList3,
+			expectFiltered: VulnerabilityList{},
+			expectSev:      models.SevNone,
+		},
+		{
+			vl: vulnList2,
+			wl: whiteList3,
+			expectFiltered: VulnerabilityList{
+				{ID: "CVE-2018-10754",
+					Severity: models.SevLow,
+					Pkg:      "ncurses",
+					Version:  "6.0+20161126-1+deb9u2",
+				},
+				{
+					ID:       "CVE-2018-6485",
+					Severity: models.SevHigh,
+					Pkg:      "glibc",
+					Version:  "2.24-11+deb9u4",
+				},
+			},
+			expectSev: models.SevNone,
+		},
+	}
+	for _, c := range cases {
+		filtered := c.vl.ApplyWhitelist(c.wl)
+		assert.Equal(t, c.expectFiltered, filtered)
+		assert.Equal(t, c.vl.Severity(), c.expectSev)
+	}
+}
+
+func TestVulnListByDigest(t *testing.T) {
+	_, err := VulnListByDigest("notexist")
+	assert.NotNil(t, err)
+}
+
+func TestVulnListFromClairResult(t *testing.T) {
+	l := VulnListFromClairResult(nil)
+	assert.Equal(t, VulnerabilityList{}, l)
+	lv := &models.ClairLayerEnvelope{
+		Layer: nil,
+		Error: nil,
+	}
+	l2 := VulnListFromClairResult(lv)
+	assert.Equal(t, VulnerabilityList{}, l2)
+}

src/replication/transfer/image/transfer.go

@@ -261,6 +261,7 @@ func (t *transfer) pullManifest(repository, reference string) (
 	t.logger.Infof("pulling the manifest of image %s:%s ...", repository, reference)
 	manifest, digest, err := t.src.PullManifest(repository, reference, []string{
 		schema1.MediaTypeManifest,
+		schema1.MediaTypeSignedManifest,
 		schema2.MediaTypeManifest,
 		manifestlist.MediaTypeManifestList,
 	})