mirror of
https://github.com/goharbor/harbor.git
synced 2024-12-24 17:47:46 +01:00
Merge branch 'master' of https://github.com/goharbor/harbor into project-quota-dev
This commit is contained in:
commit
a95f8c556d
@ -15,12 +15,11 @@
|
||||
package dao
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/astaxie/beego/orm"
|
||||
"github.com/goharbor/harbor/src/common/models"
|
||||
"github.com/goharbor/harbor/src/common/utils/log"
|
||||
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
@ -33,6 +33,23 @@ type CVEWhitelistItem struct {
|
||||
}
|
||||
|
||||
// TableName ...
|
||||
func (r *CVEWhitelist) TableName() string {
|
||||
func (c *CVEWhitelist) TableName() string {
|
||||
return "cve_whitelist"
|
||||
}
|
||||
|
||||
// CVESet returns the set of CVE id of the items in the whitelist to help filter the vulnerability list
|
||||
func (c *CVEWhitelist) CVESet() map[string]struct{} {
|
||||
r := map[string]struct{}{}
|
||||
for _, it := range c.Items {
|
||||
r[it.CVEID] = struct{}{}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// IsExpired returns whether the whitelist is expired
|
||||
func (c *CVEWhitelist) IsExpired() bool {
|
||||
if c.ExpiresAt == nil {
|
||||
return false
|
||||
}
|
||||
return time.Now().Unix() >= *c.ExpiresAt
|
||||
}
|
||||
|
72
src/common/models/cve_whitelist_test.go
Normal file
72
src/common/models/cve_whitelist_test.go
Normal file
@ -0,0 +1,72 @@
|
||||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package models
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCVEWhitelist_All(t *testing.T) {
|
||||
future := int64(4411494000)
|
||||
now := time.Now().Unix()
|
||||
cases := []struct {
|
||||
input CVEWhitelist
|
||||
cveset map[string]struct{}
|
||||
expired bool
|
||||
}{
|
||||
{
|
||||
input: CVEWhitelist{
|
||||
ID: 1,
|
||||
ProjectID: 0,
|
||||
Items: []CVEWhitelistItem{},
|
||||
},
|
||||
cveset: map[string]struct{}{},
|
||||
expired: false,
|
||||
},
|
||||
{
|
||||
input: CVEWhitelist{
|
||||
ID: 1,
|
||||
ProjectID: 0,
|
||||
Items: []CVEWhitelistItem{},
|
||||
ExpiresAt: &now,
|
||||
},
|
||||
cveset: map[string]struct{}{},
|
||||
expired: true,
|
||||
},
|
||||
{
|
||||
input: CVEWhitelist{
|
||||
ID: 2,
|
||||
ProjectID: 3,
|
||||
Items: []CVEWhitelistItem{
|
||||
{CVEID: "CVE-1999-0067"},
|
||||
{CVEID: "CVE-2016-7654321"},
|
||||
},
|
||||
ExpiresAt: &future,
|
||||
},
|
||||
cveset: map[string]struct{}{
|
||||
"CVE-1999-0067": {},
|
||||
"CVE-2016-7654321": {},
|
||||
},
|
||||
expired: false,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
assert.Equal(t, c.expired, c.input.IsExpired())
|
||||
assert.True(t, reflect.DeepEqual(c.cveset, c.input.CVESet()))
|
||||
}
|
||||
}
|
@ -34,31 +34,6 @@ type ScanJob struct {
|
||||
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
|
||||
}
|
||||
|
||||
// Severity represents the severity of a image/component in terms of vulnerability.
|
||||
type Severity int64
|
||||
|
||||
// Sevxxx is the list of severity of image after scanning.
|
||||
const (
|
||||
_ Severity = iota
|
||||
SevNone
|
||||
SevUnknown
|
||||
SevLow
|
||||
SevMedium
|
||||
SevHigh
|
||||
)
|
||||
|
||||
// String is the output function for sererity variable
|
||||
func (sev Severity) String() string {
|
||||
name := []string{"negligible", "unknown", "low", "medium", "high"}
|
||||
i := int64(sev)
|
||||
switch {
|
||||
case i >= 1 && i <= int64(SevHigh):
|
||||
return name[i-1]
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// TableName is required by by beego orm to map ScanJob to table img_scan_job
|
||||
func (s *ScanJob) TableName() string {
|
||||
return ScanJobTable
|
||||
@ -101,17 +76,6 @@ type ImageScanReq struct {
|
||||
Tag string `json:"tag"`
|
||||
}
|
||||
|
||||
// VulnerabilityItem is an item in the vulnerability result returned by vulnerability details API.
|
||||
type VulnerabilityItem struct {
|
||||
ID string `json:"id"`
|
||||
Severity Severity `json:"severity"`
|
||||
Pkg string `json:"package"`
|
||||
Version string `json:"version"`
|
||||
Description string `json:"description"`
|
||||
Link string `json:"link"`
|
||||
Fixed string `json:"fixedVersion,omitempty"`
|
||||
}
|
||||
|
||||
// ScanAllPolicy is represent the json request and object for scan all policy, the parm is het
|
||||
type ScanAllPolicy struct {
|
||||
Type string `json:"type"`
|
||||
|
26
src/common/models/sev.go
Normal file
26
src/common/models/sev.go
Normal file
@ -0,0 +1,26 @@
|
||||
package models
|
||||
|
||||
// Severity represents the severity of a image/component in terms of vulnerability.
|
||||
type Severity int64
|
||||
|
||||
// Sevxxx is the list of severity of image after scanning.
|
||||
const (
|
||||
_ Severity = iota
|
||||
SevNone
|
||||
SevUnknown
|
||||
SevLow
|
||||
SevMedium
|
||||
SevHigh
|
||||
)
|
||||
|
||||
// String is the output function for severity variable
|
||||
func (sev Severity) String() string {
|
||||
name := []string{"negligible", "unknown", "low", "medium", "high"}
|
||||
i := int64(sev)
|
||||
switch {
|
||||
case i >= 1 && i <= int64(SevHigh):
|
||||
return name[i-1]
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
@ -35,20 +35,14 @@ const googleEndpoint = "https://accounts.google.com"
|
||||
|
||||
type providerHelper struct {
|
||||
sync.Mutex
|
||||
ep endpoint
|
||||
instance atomic.Value
|
||||
setting atomic.Value
|
||||
}
|
||||
|
||||
type endpoint struct {
|
||||
url string
|
||||
VerifyCert bool
|
||||
instance atomic.Value
|
||||
setting atomic.Value
|
||||
creationTime time.Time
|
||||
}
|
||||
|
||||
func (p *providerHelper) get() (*gooidc.Provider, error) {
|
||||
if p.instance.Load() != nil {
|
||||
s := p.setting.Load().(models.OIDCSetting)
|
||||
if s.Endpoint != p.ep.url || s.VerifyCert != p.ep.VerifyCert { // relevant settings have changed, need to re-create provider.
|
||||
if time.Now().Sub(p.creationTime) > 3*time.Second {
|
||||
if err := p.create(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -57,7 +51,7 @@ func (p *providerHelper) get() (*gooidc.Provider, error) {
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
if p.instance.Load() == nil {
|
||||
if err := p.reload(); err != nil {
|
||||
if err := p.reloadSetting(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := p.create(); err != nil {
|
||||
@ -65,7 +59,7 @@ func (p *providerHelper) get() (*gooidc.Provider, error) {
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
if err := p.reload(); err != nil {
|
||||
if err := p.reloadSetting(); err != nil {
|
||||
log.Warningf("Failed to refresh configuration, error: %v", err)
|
||||
}
|
||||
time.Sleep(3 * time.Second)
|
||||
@ -73,10 +67,11 @@ func (p *providerHelper) get() (*gooidc.Provider, error) {
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
return p.instance.Load().(*gooidc.Provider), nil
|
||||
}
|
||||
|
||||
func (p *providerHelper) reload() error {
|
||||
func (p *providerHelper) reloadSetting() error {
|
||||
conf, err := config.OIDCSetting()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load OIDC setting: %v", err)
|
||||
@ -96,10 +91,7 @@ func (p *providerHelper) create() error {
|
||||
return fmt.Errorf("failed to create OIDC provider, error: %v", err)
|
||||
}
|
||||
p.instance.Store(provider)
|
||||
p.ep = endpoint{
|
||||
url: s.Endpoint,
|
||||
VerifyCert: s.VerifyCert,
|
||||
}
|
||||
p.creationTime = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -49,21 +49,20 @@ func TestMain(m *testing.M) {
|
||||
func TestHelperLoadConf(t *testing.T) {
|
||||
testP := &providerHelper{}
|
||||
assert.Nil(t, testP.setting.Load())
|
||||
err := testP.reload()
|
||||
err := testP.reloadSetting()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "test", testP.setting.Load().(models.OIDCSetting).Name)
|
||||
assert.Equal(t, endpoint{}, testP.ep)
|
||||
}
|
||||
|
||||
func TestHelperCreate(t *testing.T) {
|
||||
testP := &providerHelper{}
|
||||
err := testP.reload()
|
||||
err := testP.reloadSetting()
|
||||
assert.Nil(t, err)
|
||||
assert.Nil(t, testP.instance.Load())
|
||||
err = testP.create()
|
||||
assert.Nil(t, err)
|
||||
assert.EqualValues(t, "https://accounts.google.com", testP.ep.url)
|
||||
assert.NotNil(t, testP.instance.Load())
|
||||
assert.True(t, time.Now().Sub(testP.creationTime) < 2*time.Second)
|
||||
}
|
||||
|
||||
func TestHelperGet(t *testing.T) {
|
||||
|
@ -17,6 +17,7 @@ package api
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/goharbor/harbor/src/pkg/scan"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"sort"
|
||||
@ -34,7 +35,6 @@ import (
|
||||
"github.com/goharbor/harbor/src/common/models"
|
||||
"github.com/goharbor/harbor/src/common/rbac"
|
||||
"github.com/goharbor/harbor/src/common/utils"
|
||||
"github.com/goharbor/harbor/src/common/utils/clair"
|
||||
"github.com/goharbor/harbor/src/common/utils/log"
|
||||
"github.com/goharbor/harbor/src/common/utils/notary"
|
||||
"github.com/goharbor/harbor/src/common/utils/registry"
|
||||
@ -1036,21 +1036,9 @@ func (ra *RepositoryAPI) VulnerabilityDetails() {
|
||||
ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
|
||||
return
|
||||
}
|
||||
res := []*models.VulnerabilityItem{}
|
||||
overview, err := dao.GetImgScanOverview(digest)
|
||||
res, err := scan.VulnListByDigest(digest)
|
||||
if err != nil {
|
||||
ra.SendInternalServerError(fmt.Errorf("failed to get the scan overview, error: %v", err))
|
||||
return
|
||||
}
|
||||
if overview != nil && len(overview.DetailsKey) > 0 {
|
||||
clairClient := clair.NewClient(config.ClairEndpoint(), nil)
|
||||
log.Debugf("The key for getting details: %s", overview.DetailsKey)
|
||||
details, err := clairClient.GetResult(overview.DetailsKey)
|
||||
if err != nil {
|
||||
ra.SendInternalServerError(fmt.Errorf("Failed to get scan details from Clair, error: %v", err))
|
||||
return
|
||||
}
|
||||
res = transformVulnerabilities(details)
|
||||
log.Errorf("Failed to get vulnerability list for image: %s:%s", repository, tag)
|
||||
}
|
||||
ra.Data["json"] = res
|
||||
ra.ServeJSON()
|
||||
|
@ -24,7 +24,6 @@ import (
|
||||
commonhttp "github.com/goharbor/harbor/src/common/http"
|
||||
"github.com/goharbor/harbor/src/common/models"
|
||||
"github.com/goharbor/harbor/src/common/utils"
|
||||
"github.com/goharbor/harbor/src/common/utils/clair"
|
||||
"github.com/goharbor/harbor/src/common/utils/log"
|
||||
"github.com/goharbor/harbor/src/common/utils/registry"
|
||||
"github.com/goharbor/harbor/src/common/utils/registry/auth"
|
||||
@ -279,35 +278,3 @@ func repositoryExist(name string, client *registry.Repository) (bool, error) {
|
||||
}
|
||||
return len(tags) != 0, nil
|
||||
}
|
||||
|
||||
// transformVulnerabilities transforms the returned value of Clair API to a list of VulnerabilityItem
|
||||
func transformVulnerabilities(layerWithVuln *models.ClairLayerEnvelope) []*models.VulnerabilityItem {
|
||||
res := []*models.VulnerabilityItem{}
|
||||
l := layerWithVuln.Layer
|
||||
if l == nil {
|
||||
return res
|
||||
}
|
||||
features := l.Features
|
||||
if features == nil {
|
||||
return res
|
||||
}
|
||||
for _, f := range features {
|
||||
vulnerabilities := f.Vulnerabilities
|
||||
if vulnerabilities == nil {
|
||||
continue
|
||||
}
|
||||
for _, v := range vulnerabilities {
|
||||
vItem := &models.VulnerabilityItem{
|
||||
ID: v.Name,
|
||||
Pkg: f.Name,
|
||||
Version: f.Version,
|
||||
Severity: clair.ParseClairSev(v.Severity),
|
||||
Fixed: v.FixedBy,
|
||||
Link: v.Link,
|
||||
Description: v.Description,
|
||||
}
|
||||
res = append(res, vItem)
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
@ -229,8 +229,10 @@ type oidcCliReqCtxModifier struct{}
|
||||
|
||||
func (oc *oidcCliReqCtxModifier) Modify(ctx *beegoctx.Context) bool {
|
||||
path := ctx.Request.URL.Path
|
||||
if path != "/service/token" && !strings.HasPrefix(path, "/chartrepo/") {
|
||||
log.Debug("OIDC CLI modifer only handles request by docker CLI or helm CLI")
|
||||
if path != "/service/token" &&
|
||||
!strings.HasPrefix(path, "/chartrepo/") &&
|
||||
!strings.HasPrefix(path, "/api/chartrepo/") {
|
||||
log.Debug("OIDC CLI modifier only handles request by docker CLI or helm CLI")
|
||||
return false
|
||||
}
|
||||
if ctx.Request.Context().Value(AuthModeKey).(string) != common.OIDCAuth {
|
||||
|
@ -2,7 +2,6 @@ package proxy
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/dao"
|
||||
"github.com/goharbor/harbor/src/common/models"
|
||||
"github.com/goharbor/harbor/src/common/utils/clair"
|
||||
@ -11,6 +10,7 @@ import (
|
||||
"github.com/goharbor/harbor/src/core/config"
|
||||
"github.com/goharbor/harbor/src/core/promgr"
|
||||
coreutils "github.com/goharbor/harbor/src/core/utils"
|
||||
"github.com/goharbor/harbor/src/pkg/scan"
|
||||
|
||||
"context"
|
||||
"fmt"
|
||||
@ -303,27 +303,41 @@ func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request)
|
||||
vh.next.ServeHTTP(rw, req)
|
||||
return
|
||||
}
|
||||
overview, err := dao.GetImgScanOverview(img.digest)
|
||||
// TODO: Get whitelist based on project setting
|
||||
wl, err := dao.GetSysCVEWhitelist()
|
||||
if err != nil {
|
||||
log.Errorf("failed to get ImgScanOverview with repo: %s, reference: %s, digest: %s. Error: %v", img.repository, img.reference, img.digest, err)
|
||||
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get ImgScanOverview."), http.StatusPreconditionFailed)
|
||||
log.Errorf("Failed to get the whitelist, error: %v", err)
|
||||
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get CVE whitelist."), http.StatusPreconditionFailed)
|
||||
return
|
||||
}
|
||||
// severity is 0 means that the image fails to scan or not scanned successfully.
|
||||
if overview == nil || overview.Sev == 0 {
|
||||
log.Debugf("cannot get the image scan overview info, failing the response.")
|
||||
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Cannot get the image severity."), http.StatusPreconditionFailed)
|
||||
vl, err := scan.VulnListByDigest(img.digest)
|
||||
if err != nil {
|
||||
log.Errorf("Failed to get the vulnerability list, error: %v", err)
|
||||
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get vulnerabilities."), http.StatusPreconditionFailed)
|
||||
return
|
||||
}
|
||||
imageSev := overview.Sev
|
||||
if imageSev >= int(projectVulnerableSeverity) {
|
||||
log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", models.Severity(imageSev), projectVulnerableSeverity)
|
||||
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", models.Severity(imageSev), projectVulnerableSeverity)), http.StatusPreconditionFailed)
|
||||
filtered := vl.ApplyWhitelist(*wl)
|
||||
msg := vh.filterMsg(img, filtered)
|
||||
log.Info(msg)
|
||||
if int(vl.Severity()) >= int(projectVulnerableSeverity) {
|
||||
log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", vl.Severity(), projectVulnerableSeverity)
|
||||
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", vl.Severity(), projectVulnerableSeverity)), http.StatusPreconditionFailed)
|
||||
return
|
||||
}
|
||||
vh.next.ServeHTTP(rw, req)
|
||||
}
|
||||
|
||||
func (vh vulnerableHandler) filterMsg(img imageInfo, filtered scan.VulnerabilityList) string {
|
||||
filterMsg := fmt.Sprintf("Image: %s/%s:%s, digest: %s, vulnerabilities fitered by whitelist:", img.projectName, img.repository, img.reference, img.digest)
|
||||
if len(filtered) == 0 {
|
||||
filterMsg = fmt.Sprintf("%s none.", filterMsg)
|
||||
}
|
||||
for _, v := range filtered {
|
||||
filterMsg = fmt.Sprintf("%s ID: %s, severity: %s;", filterMsg, v.ID, v.Severity)
|
||||
}
|
||||
return filterMsg
|
||||
}
|
||||
|
||||
func matchNotaryDigest(img imageInfo) (bool, error) {
|
||||
if NotaryEndpoint == "" {
|
||||
NotaryEndpoint = config.InternalNotaryEndpoint()
|
||||
|
@ -38,7 +38,15 @@ func Init(urls ...string) error {
|
||||
return err
|
||||
}
|
||||
Proxy = httputil.NewSingleHostReverseProxy(targetURL)
|
||||
handlers = handlerChain{head: readonlyHandler{next: urlHandler{next: multipleManifestHandler{next: listReposHandler{next: contentTrustHandler{next: vulnerableHandler{next: Proxy}}}}}}}
|
||||
handlers = handlerChain{
|
||||
head: readonlyHandler{
|
||||
next: urlHandler{
|
||||
next: multipleManifestHandler{
|
||||
next: listReposHandler{
|
||||
next: contentTrustHandler{
|
||||
next: vulnerableHandler{
|
||||
next: Proxy,
|
||||
}}}}}}}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -34,7 +34,7 @@ func RedisKeyScheduled(namespace string) string {
|
||||
|
||||
// RedisKeyLastPeriodicEnqueue returns key of timestamp if last periodic enqueue.
|
||||
func RedisKeyLastPeriodicEnqueue(namespace string) string {
|
||||
return RedisNamespacePrefix(namespace) + "last_periodic_enqueue"
|
||||
return RedisNamespacePrefix(namespace) + "last_periodic_enqueue_h"
|
||||
}
|
||||
|
||||
// KeyNamespacePrefix returns the based key based on the namespace.
|
||||
|
@ -16,6 +16,12 @@ package period
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/goharbor/harbor/src/jobservice/common/rds"
|
||||
"github.com/goharbor/harbor/src/jobservice/common/utils"
|
||||
"github.com/goharbor/harbor/src/jobservice/env"
|
||||
@ -26,9 +32,6 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// EnqueuerTestSuite tests functions of enqueuer
|
||||
@ -89,19 +92,30 @@ func (suite *EnqueuerTestSuite) TestEnqueuer() {
|
||||
suite.enqueuer.stopChan <- true
|
||||
}()
|
||||
|
||||
<-time.After(1 * time.Second)
|
||||
|
||||
key := rds.RedisKeyScheduled(suite.namespace)
|
||||
conn := suite.pool.Get()
|
||||
defer func() {
|
||||
_ = conn.Close()
|
||||
}()
|
||||
|
||||
count, err := redis.Int(conn.Do("ZCARD", key))
|
||||
require.Nil(suite.T(), err, "count scheduled: nil error expected but got %s", err)
|
||||
assert.Condition(suite.T(), func() bool {
|
||||
return count > 0
|
||||
}, "count of scheduled jobs should be greater than 0 but got %d", count)
|
||||
tk := time.NewTicker(500 * time.Millisecond)
|
||||
defer tk.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-tk.C:
|
||||
count, err := redis.Int(conn.Do("ZCARD", key))
|
||||
require.Nil(suite.T(), err, "count scheduled: nil error expected but got %s", err)
|
||||
if assert.Condition(suite.T(), func() (success bool) {
|
||||
return count > 0
|
||||
}, "at least one job should be scheduled for the periodic job policy") {
|
||||
return
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
require.NoError(suite.T(), errors.New("timeout (15s): expect at 1 scheduled job but still get nothing"))
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
err := suite.enqueuer.start()
|
||||
@ -112,7 +126,7 @@ func (suite *EnqueuerTestSuite) prepare() {
|
||||
now := time.Now()
|
||||
minute := now.Minute()
|
||||
|
||||
coreSpec := fmt.Sprintf("30,50 %d * * * *", minute+2)
|
||||
coreSpec := fmt.Sprintf("0-59 %d * * * *", minute)
|
||||
|
||||
// Prepare one
|
||||
p := &Policy{
|
||||
|
136
src/pkg/scan/vuln.go
Normal file
136
src/pkg/scan/vuln.go
Normal file
@ -0,0 +1,136 @@
|
||||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package scan
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/goharbor/harbor/src/common/dao"
|
||||
"github.com/goharbor/harbor/src/common/models"
|
||||
"github.com/goharbor/harbor/src/common/utils/clair"
|
||||
"github.com/goharbor/harbor/src/common/utils/log"
|
||||
"github.com/goharbor/harbor/src/core/config"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
// VulnerabilityItem represents a vulnerability reported by scanner
|
||||
type VulnerabilityItem struct {
|
||||
ID string `json:"id"`
|
||||
Severity models.Severity `json:"severity"`
|
||||
Pkg string `json:"package"`
|
||||
Version string `json:"version"`
|
||||
Description string `json:"description"`
|
||||
Link string `json:"link"`
|
||||
Fixed string `json:"fixedVersion,omitempty"`
|
||||
}
|
||||
|
||||
// VulnerabilityList is a list of vulnerabilities, which should be scanner-agnostic
|
||||
type VulnerabilityList []VulnerabilityItem
|
||||
|
||||
// ApplyWhitelist filters out the CVE defined in the whitelist in the parm.
|
||||
// It returns the items that are filtered for the caller to track or log.
|
||||
func (vl *VulnerabilityList) ApplyWhitelist(whitelist models.CVEWhitelist) VulnerabilityList {
|
||||
filtered := VulnerabilityList{}
|
||||
if whitelist.IsExpired() {
|
||||
log.Info("The input whitelist is expired, skip filtering")
|
||||
return filtered
|
||||
}
|
||||
s := whitelist.CVESet()
|
||||
r := (*vl)[:0]
|
||||
for _, v := range *vl {
|
||||
if _, ok := s[v.ID]; ok {
|
||||
log.Debugf("Filtered Vulnerability in whitelist, CVE ID: %s, severity: %s", v.ID, v.Severity)
|
||||
filtered = append(filtered, v)
|
||||
} else {
|
||||
r = append(r, v)
|
||||
}
|
||||
}
|
||||
val := reflect.ValueOf(vl)
|
||||
val.Elem().SetLen(len(r))
|
||||
return filtered
|
||||
}
|
||||
|
||||
// Severity returns the highest severity of the vulnerabilities in the list
|
||||
func (vl *VulnerabilityList) Severity() models.Severity {
|
||||
s := models.SevNone
|
||||
for _, v := range *vl {
|
||||
if v.Severity > s {
|
||||
s = v.Severity
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// HasCVE returns whether the vulnerability list has the vulnerability with CVE ID in the parm
|
||||
func (vl *VulnerabilityList) HasCVE(id string) bool {
|
||||
for _, v := range *vl {
|
||||
if v.ID == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// VulnListFromClairResult transforms the returned value of Clair API to a VulnerabilityList
|
||||
func VulnListFromClairResult(layerWithVuln *models.ClairLayerEnvelope) VulnerabilityList {
|
||||
res := VulnerabilityList{}
|
||||
if layerWithVuln == nil {
|
||||
return res
|
||||
}
|
||||
l := layerWithVuln.Layer
|
||||
if l == nil {
|
||||
return res
|
||||
}
|
||||
features := l.Features
|
||||
if features == nil {
|
||||
return res
|
||||
}
|
||||
for _, f := range features {
|
||||
vulnerabilities := f.Vulnerabilities
|
||||
if vulnerabilities == nil {
|
||||
continue
|
||||
}
|
||||
for _, v := range vulnerabilities {
|
||||
vItem := VulnerabilityItem{
|
||||
ID: v.Name,
|
||||
Pkg: f.Name,
|
||||
Version: f.Version,
|
||||
Severity: clair.ParseClairSev(v.Severity),
|
||||
Fixed: v.FixedBy,
|
||||
Link: v.Link,
|
||||
Description: v.Description,
|
||||
}
|
||||
res = append(res, vItem)
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// VulnListByDigest returns the VulnerabilityList based on the scan result of artifact with the digest in the parm
|
||||
func VulnListByDigest(digest string) (VulnerabilityList, error) {
|
||||
var res VulnerabilityList
|
||||
overview, err := dao.GetImgScanOverview(digest)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
if overview == nil || len(overview.DetailsKey) == 0 {
|
||||
return res, fmt.Errorf("unable to get the scan result for digest: %s, the artifact is not scanned", digest)
|
||||
}
|
||||
c := clair.NewClient(config.ClairEndpoint(), nil)
|
||||
clairRes, err := c.GetResult(overview.DetailsKey)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("failed to get scan result from Clair, error: %v", err)
|
||||
}
|
||||
return VulnListFromClairResult(clairRes), nil
|
||||
}
|
178
src/pkg/scan/vuln_test.go
Normal file
178
src/pkg/scan/vuln_test.go
Normal file
@ -0,0 +1,178 @@
|
||||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package scan
|
||||
|
||||
import (
|
||||
"github.com/goharbor/harbor/src/common/dao"
|
||||
"github.com/goharbor/harbor/src/common/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var (
|
||||
past = int64(1561967574)
|
||||
vulnList1 = VulnerabilityList{}
|
||||
vulnList2 = VulnerabilityList{
|
||||
{ID: "CVE-2018-10754",
|
||||
Severity: models.SevLow,
|
||||
Pkg: "ncurses",
|
||||
Version: "6.0+20161126-1+deb9u2",
|
||||
},
|
||||
{
|
||||
ID: "CVE-2018-6485",
|
||||
Severity: models.SevHigh,
|
||||
Pkg: "glibc",
|
||||
Version: "2.24-11+deb9u4",
|
||||
},
|
||||
}
|
||||
whiteList1 = models.CVEWhitelist{
|
||||
ExpiresAt: &past,
|
||||
Items: []models.CVEWhitelistItem{
|
||||
{CVEID: "CVE-2018-6485"},
|
||||
},
|
||||
}
|
||||
whiteList2 = models.CVEWhitelist{
|
||||
Items: []models.CVEWhitelistItem{
|
||||
{CVEID: "CVE-2018-6485"},
|
||||
},
|
||||
}
|
||||
whiteList3 = models.CVEWhitelist{
|
||||
Items: []models.CVEWhitelistItem{
|
||||
{CVEID: "CVE-2018-6485"},
|
||||
{CVEID: "CVE-2018-10754"},
|
||||
{CVEID: "CVE-2019-12817"},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
dao.PrepareTestForPostgresSQL()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func TestVulnerabilityList_HasCVE(t *testing.T) {
|
||||
cases := []struct {
|
||||
input VulnerabilityList
|
||||
cve string
|
||||
result bool
|
||||
}{
|
||||
{
|
||||
input: vulnList1,
|
||||
cve: "CVE-2018-10754",
|
||||
result: false,
|
||||
},
|
||||
{
|
||||
input: vulnList2,
|
||||
cve: "CVE-2018-10754",
|
||||
result: true,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
assert.Equal(t, c.result, c.input.HasCVE(c.cve))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVulnerabilityList_Severity(t *testing.T) {
|
||||
cases := []struct {
|
||||
input VulnerabilityList
|
||||
expect models.Severity
|
||||
}{
|
||||
{
|
||||
input: vulnList1,
|
||||
expect: models.SevNone,
|
||||
}, {
|
||||
input: vulnList2,
|
||||
expect: models.SevHigh,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
assert.Equal(t, c.expect, c.input.Severity())
|
||||
}
|
||||
}
|
||||
|
||||
func TestVulnerabilityList_ApplyWhitelist(t *testing.T) {
|
||||
cases := []struct {
|
||||
vl VulnerabilityList
|
||||
wl models.CVEWhitelist
|
||||
expectFiltered VulnerabilityList
|
||||
expectSev models.Severity
|
||||
}{
|
||||
{
|
||||
vl: vulnList2,
|
||||
wl: whiteList1,
|
||||
expectFiltered: VulnerabilityList{},
|
||||
expectSev: models.SevHigh,
|
||||
},
|
||||
{
|
||||
vl: vulnList2,
|
||||
wl: whiteList2,
|
||||
expectFiltered: VulnerabilityList{
|
||||
{
|
||||
ID: "CVE-2018-6485",
|
||||
Severity: models.SevHigh,
|
||||
Pkg: "glibc",
|
||||
Version: "2.24-11+deb9u4",
|
||||
},
|
||||
},
|
||||
expectSev: models.SevLow,
|
||||
},
|
||||
{
|
||||
vl: vulnList1,
|
||||
wl: whiteList3,
|
||||
expectFiltered: VulnerabilityList{},
|
||||
expectSev: models.SevNone,
|
||||
},
|
||||
{
|
||||
vl: vulnList2,
|
||||
wl: whiteList3,
|
||||
expectFiltered: VulnerabilityList{
|
||||
{ID: "CVE-2018-10754",
|
||||
Severity: models.SevLow,
|
||||
Pkg: "ncurses",
|
||||
Version: "6.0+20161126-1+deb9u2",
|
||||
},
|
||||
{
|
||||
ID: "CVE-2018-6485",
|
||||
Severity: models.SevHigh,
|
||||
Pkg: "glibc",
|
||||
Version: "2.24-11+deb9u4",
|
||||
},
|
||||
},
|
||||
expectSev: models.SevNone,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
filtered := c.vl.ApplyWhitelist(c.wl)
|
||||
assert.Equal(t, c.expectFiltered, filtered)
|
||||
assert.Equal(t, c.vl.Severity(), c.expectSev)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVulnListByDigest(t *testing.T) {
|
||||
_, err := VulnListByDigest("notexist")
|
||||
assert.NotNil(t, err)
|
||||
}
|
||||
|
||||
func TestVulnListFromClairResult(t *testing.T) {
|
||||
l := VulnListFromClairResult(nil)
|
||||
assert.Equal(t, VulnerabilityList{}, l)
|
||||
lv := &models.ClairLayerEnvelope{
|
||||
Layer: nil,
|
||||
Error: nil,
|
||||
}
|
||||
l2 := VulnListFromClairResult(lv)
|
||||
assert.Equal(t, VulnerabilityList{}, l2)
|
||||
}
|
@ -261,6 +261,7 @@ func (t *transfer) pullManifest(repository, reference string) (
|
||||
t.logger.Infof("pulling the manifest of image %s:%s ...", repository, reference)
|
||||
manifest, digest, err := t.src.PullManifest(repository, reference, []string{
|
||||
schema1.MediaTypeManifest,
|
||||
schema1.MediaTypeSignedManifest,
|
||||
schema2.MediaTypeManifest,
|
||||
manifestlist.MediaTypeManifestList,
|
||||
})
|
||||
|
Loading…
Reference in New Issue
Block a user