Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-07-02 17:55:15 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #8179 from reasonerjt/interceptor-use-whitelist

Browse Source 
Apply CVE white list in interceptor
This commit is contained in:
Daniel Jiang 2019-07-03 15:12:33 +08:00 committed by GitHub
commit 5d887ad0d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 470 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/common/dao/scan_job.go
View File

@ -15,12 +15,11 @@
package dao
import (
"encoding/json"
"fmt"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"encoding/json"
"fmt"
"time"
)

19
src/common/models/cve_whitelist.go
View File

@ -33,6 +33,23 @@ type CVEWhitelistItem struct {
}
// TableName ...
func (r *CVEWhitelist) TableName() string {
func (c *CVEWhitelist) TableName() string {
return "cve_whitelist"
}
// CVESet returns the set of CVE id of the items in the whitelist to help filter the vulnerability list
func (c *CVEWhitelist) CVESet() map[string]struct{} {
r := map[string]struct{}{}
for _, it := range c.Items {
r[it.CVEID] = struct{}{}
}
return r
}
// IsExpired returns whether the whitelist is expired
func (c *CVEWhitelist) IsExpired() bool {
if c.ExpiresAt == nil {
return false
}
return time.Now().Unix() >= *c.ExpiresAt
}

72
src/common/models/cve_whitelist_test.go Normal file
View File

@ -0,0 +1,72 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"github.com/stretchr/testify/assert"
"reflect"
"testing"
"time"
)
func TestCVEWhitelist_All(t *testing.T) {
future := int64(4411494000)
now := time.Now().Unix()
cases := []struct {
input CVEWhitelist
cveset map[string]struct{}
expired bool
}{
{
input: CVEWhitelist{
ID: 1,
ProjectID: 0,
Items: []CVEWhitelistItem{},
},
cveset: map[string]struct{}{},
expired: false,
},
{
input: CVEWhitelist{
ID: 1,
ProjectID: 0,
Items: []CVEWhitelistItem{},
ExpiresAt: &now,
},
cveset: map[string]struct{}{},
expired: true,
},
{
input: CVEWhitelist{
ID: 2,
ProjectID: 3,
Items: []CVEWhitelistItem{
{CVEID: "CVE-1999-0067"},
{CVEID: "CVE-2016-7654321"},
},
ExpiresAt: &future,
},
cveset: map[string]struct{}{
"CVE-1999-0067": {},
"CVE-2016-7654321": {},
},
expired: false,
},
}
for _, c := range cases {
assert.Equal(t, c.expired, c.input.IsExpired())
assert.True(t, reflect.DeepEqual(c.cveset, c.input.CVESet()))
}
}

36
src/common/models/scan_job.go
View File

@ -34,31 +34,6 @@ type ScanJob struct {
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
// Severity represents the severity of a image/component in terms of vulnerability.
type Severity int64
// Sevxxx is the list of severity of image after scanning.
const (
_ Severity = iota
SevNone
SevUnknown
SevLow
SevMedium
SevHigh
)
// String is the output function for sererity variable
func (sev Severity) String() string {
name := []string{"negligible", "unknown", "low", "medium", "high"}
i := int64(sev)
switch {
case i >= 1 && i <= int64(SevHigh):
return name[i-1]
default:
return "unknown"
}
}
// TableName is required by by beego orm to map ScanJob to table img_scan_job
func (s *ScanJob) TableName() string {
return ScanJobTable
@ -101,17 +76,6 @@ type ImageScanReq struct {
Tag string `json:"tag"`
}
// VulnerabilityItem is an item in the vulnerability result returned by vulnerability details API.
type VulnerabilityItem struct {
ID string `json:"id"`
Severity Severity `json:"severity"`
Pkg string `json:"package"`
Version string `json:"version"`
Description string `json:"description"`
Link string `json:"link"`
Fixed string `json:"fixedVersion,omitempty"`
}
// ScanAllPolicy is represent the json request and object for scan all policy, the parm is het
type ScanAllPolicy struct {
Type string `json:"type"`

26
src/common/models/sev.go Normal file
View File

@ -0,0 +1,26 @@
package models
// Severity represents the severity of a image/component in terms of vulnerability.
type Severity int64
// Sevxxx is the list of severity of image after scanning.
const (
_ Severity = iota
SevNone
SevUnknown
SevLow
SevMedium
SevHigh
)
// String is the output function for severity variable
func (sev Severity) String() string {
name := []string{"negligible", "unknown", "low", "medium", "high"}
i := int64(sev)
switch {
case i >= 1 && i <= int64(SevHigh):
return name[i-1]
default:
return "unknown"
}
}

18
src/core/api/repository.go
View File

@ -17,6 +17,7 @@ package api
import (
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/pkg/scan"
"io/ioutil"
"net/http"
"sort"
@ -34,7 +35,6 @@ import (
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/clair"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/notary"
"github.com/goharbor/harbor/src/common/utils/registry"
@ -1036,21 +1036,9 @@ func (ra *RepositoryAPI) VulnerabilityDetails() {
ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
return
}
res := []*models.VulnerabilityItem{}
overview, err := dao.GetImgScanOverview(digest)
res, err := scan.VulnListByDigest(digest)
if err != nil {
ra.SendInternalServerError(fmt.Errorf("failed to get the scan overview, error: %v", err))
return
}
if overview != nil && len(overview.DetailsKey) > 0 {
clairClient := clair.NewClient(config.ClairEndpoint(), nil)
log.Debugf("The key for getting details: %s", overview.DetailsKey)
details, err := clairClient.GetResult(overview.DetailsKey)
if err != nil {
ra.SendInternalServerError(fmt.Errorf("Failed to get scan details from Clair, error: %v", err))
return
}
res = transformVulnerabilities(details)
log.Errorf("Failed to get vulnerability list for image: %s:%s", repository, tag)
}
ra.Data["json"] = res
ra.ServeJSON()

33
src/core/api/utils.go
View File

@ -24,7 +24,6 @@ import (
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/clair"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
@ -279,35 +278,3 @@ func repositoryExist(name string, client *registry.Repository) (bool, error) {
}
return len(tags) != 0, nil
}
// transformVulnerabilities transforms the returned value of Clair API to a list of VulnerabilityItem
func transformVulnerabilities(layerWithVuln *models.ClairLayerEnvelope) []*models.VulnerabilityItem {
res := []*models.VulnerabilityItem{}
l := layerWithVuln.Layer
if l == nil {
return res
}
features := l.Features
if features == nil {
return res
}
for _, f := range features {
vulnerabilities := f.Vulnerabilities
if vulnerabilities == nil {
continue
}
for _, v := range vulnerabilities {
vItem := &models.VulnerabilityItem{
ID: v.Name,
Pkg: f.Name,
Version: f.Version,
Severity: clair.ParseClairSev(v.Severity),
Fixed: v.FixedBy,
Link: v.Link,
Description: v.Description,
}
res = append(res, vItem)
}
}
return res
}

38
src/core/proxy/interceptors.go
View File

@ -2,7 +2,6 @@ package proxy
import (
"encoding/json"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/clair"
@ -11,6 +10,7 @@ import (
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/promgr"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/pkg/scan"
"context"
"fmt"
@ -303,27 +303,41 @@ func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request)
vh.next.ServeHTTP(rw, req)
return
}
overview, err := dao.GetImgScanOverview(img.digest)
// TODO: Get whitelist based on project setting
wl, err := dao.GetSysCVEWhitelist()
if err != nil {
log.Errorf("failed to get ImgScanOverview with repo: %s, reference: %s, digest: %s. Error: %v", img.repository, img.reference, img.digest, err)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get ImgScanOverview."), http.StatusPreconditionFailed)
log.Errorf("Failed to get the whitelist, error: %v", err)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get CVE whitelist."), http.StatusPreconditionFailed)
return
}
// severity is 0 means that the image fails to scan or not scanned successfully.
if overview == nil || overview.Sev == 0 {
log.Debugf("cannot get the image scan overview info, failing the response.")
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Cannot get the image severity."), http.StatusPreconditionFailed)
vl, err := scan.VulnListByDigest(img.digest)
if err != nil {
log.Errorf("Failed to get the vulnerability list, error: %v", err)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get vulnerabilities."), http.StatusPreconditionFailed)
return
}
imageSev := overview.Sev
if imageSev >= int(projectVulnerableSeverity) {
log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", models.Severity(imageSev), projectVulnerableSeverity)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", models.Severity(imageSev), projectVulnerableSeverity)), http.StatusPreconditionFailed)
filtered := vl.ApplyWhitelist(*wl)
msg := vh.filterMsg(img, filtered)
log.Info(msg)
if int(vl.Severity()) >= int(projectVulnerableSeverity) {
log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", vl.Severity(), projectVulnerableSeverity)
http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", vl.Severity(), projectVulnerableSeverity)), http.StatusPreconditionFailed)
return
}
vh.next.ServeHTTP(rw, req)
}
func (vh vulnerableHandler) filterMsg(img imageInfo, filtered scan.VulnerabilityList) string {
filterMsg := fmt.Sprintf("Image: %s/%s:%s, digest: %s, vulnerabilities fitered by whitelist:", img.projectName, img.repository, img.reference, img.digest)
if len(filtered) == 0 {
filterMsg = fmt.Sprintf("%s none.", filterMsg)
}
for _, v := range filtered {
filterMsg = fmt.Sprintf("%s ID: %s, severity: %s;", filterMsg, v.ID, v.Severity)
}
return filterMsg
}
func matchNotaryDigest(img imageInfo) (bool, error) {
if NotaryEndpoint == "" {
NotaryEndpoint = config.InternalNotaryEndpoint()

10
src/core/proxy/proxy.go
View File

@ -38,7 +38,15 @@ func Init(urls ...string) error {
return err
}
Proxy = httputil.NewSingleHostReverseProxy(targetURL)
handlers = handlerChain{head: readonlyHandler{next: urlHandler{next: multipleManifestHandler{next: listReposHandler{next: contentTrustHandler{next: vulnerableHandler{next: Proxy}}}}}}}
handlers = handlerChain{
head: readonlyHandler{
next: urlHandler{
next: multipleManifestHandler{
next: listReposHandler{
next: contentTrustHandler{
next: vulnerableHandler{
next: Proxy,
}}}}}}}
return nil
}

136
src/pkg/scan/vuln.go Normal file
View File

@ -0,0 +1,136 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"fmt"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/clair"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"reflect"
)
// VulnerabilityItem represents a vulnerability reported by scanner
type VulnerabilityItem struct {
ID string `json:"id"`
Severity models.Severity `json:"severity"`
Pkg string `json:"package"`
Version string `json:"version"`
Description string `json:"description"`
Link string `json:"link"`
Fixed string `json:"fixedVersion,omitempty"`
}
// VulnerabilityList is a list of vulnerabilities, which should be scanner-agnostic
type VulnerabilityList []VulnerabilityItem
// ApplyWhitelist filters out the CVE defined in the whitelist in the parm.
// It returns the items that are filtered for the caller to track or log.
func (vl *VulnerabilityList) ApplyWhitelist(whitelist models.CVEWhitelist) VulnerabilityList {
filtered := VulnerabilityList{}
if whitelist.IsExpired() {
log.Info("The input whitelist is expired, skip filtering")
return filtered
}
s := whitelist.CVESet()
r := (*vl)[:0]
for _, v := range *vl {
if _, ok := s[v.ID]; ok {
log.Debugf("Filtered Vulnerability in whitelist, CVE ID: %s, severity: %s", v.ID, v.Severity)
filtered = append(filtered, v)
} else {
r = append(r, v)
}
}
val := reflect.ValueOf(vl)
val.Elem().SetLen(len(r))
return filtered
}
// Severity returns the highest severity of the vulnerabilities in the list
func (vl *VulnerabilityList) Severity() models.Severity {
s := models.SevNone
for _, v := range *vl {
if v.Severity > s {
s = v.Severity
}
}
return s
}
// HasCVE returns whether the vulnerability list has the vulnerability with CVE ID in the parm
func (vl *VulnerabilityList) HasCVE(id string) bool {
for _, v := range *vl {
if v.ID == id {
return true
}
}
return false
}
// VulnListFromClairResult transforms the returned value of Clair API to a VulnerabilityList
func VulnListFromClairResult(layerWithVuln *models.ClairLayerEnvelope) VulnerabilityList {
res := VulnerabilityList{}
if layerWithVuln == nil {
return res
}
l := layerWithVuln.Layer
if l == nil {
return res
}
features := l.Features
if features == nil {
return res
}
for _, f := range features {
vulnerabilities := f.Vulnerabilities
if vulnerabilities == nil {
continue
}
for _, v := range vulnerabilities {
vItem := VulnerabilityItem{
ID: v.Name,
Pkg: f.Name,
Version: f.Version,
Severity: clair.ParseClairSev(v.Severity),
Fixed: v.FixedBy,
Link: v.Link,
Description: v.Description,
}
res = append(res, vItem)
}
}
return res
}
// VulnListByDigest returns the VulnerabilityList based on the scan result of artifact with the digest in the parm
func VulnListByDigest(digest string) (VulnerabilityList, error) {
var res VulnerabilityList
overview, err := dao.GetImgScanOverview(digest)
if err != nil {
return res, err
}
if overview == nil || len(overview.DetailsKey) == 0 {
return res, fmt.Errorf("unable to get the scan result for digest: %s, the artifact is not scanned", digest)
}
c := clair.NewClient(config.ClairEndpoint(), nil)
clairRes, err := c.GetResult(overview.DetailsKey)
if err != nil {
return res, fmt.Errorf("failed to get scan result from Clair, error: %v", err)
}
return VulnListFromClairResult(clairRes), nil
}

178
src/pkg/scan/vuln_test.go Normal file
View File

@ -0,0 +1,178 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package scan
import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/stretchr/testify/assert"
"os"
"testing"
)
var (
past = int64(1561967574)
vulnList1 = VulnerabilityList{}
vulnList2 = VulnerabilityList{
{ID: "CVE-2018-10754",
Severity: models.SevLow,
Pkg: "ncurses",
Version: "6.0+20161126-1+deb9u2",
},
{
ID: "CVE-2018-6485",
Severity: models.SevHigh,
Pkg: "glibc",
Version: "2.24-11+deb9u4",
},
}
whiteList1 = models.CVEWhitelist{
ExpiresAt: &past,
Items: []models.CVEWhitelistItem{
{CVEID: "CVE-2018-6485"},
},
}
whiteList2 = models.CVEWhitelist{
Items: []models.CVEWhitelistItem{
{CVEID: "CVE-2018-6485"},
},
}
whiteList3 = models.CVEWhitelist{
Items: []models.CVEWhitelistItem{
{CVEID: "CVE-2018-6485"},
{CVEID: "CVE-2018-10754"},
{CVEID: "CVE-2019-12817"},
},
}
)
func TestMain(m *testing.M) {
dao.PrepareTestForPostgresSQL()
os.Exit(m.Run())
}
func TestVulnerabilityList_HasCVE(t *testing.T) {
cases := []struct {
input VulnerabilityList
cve string
result bool
}{
{
input: vulnList1,
cve: "CVE-2018-10754",
result: false,
},
{
input: vulnList2,
cve: "CVE-2018-10754",
result: true,
},
}
for _, c := range cases {
assert.Equal(t, c.result, c.input.HasCVE(c.cve))
}
}
func TestVulnerabilityList_Severity(t *testing.T) {
cases := []struct {
input VulnerabilityList
expect models.Severity
}{
{
input: vulnList1,
expect: models.SevNone,
}, {
input: vulnList2,
expect: models.SevHigh,
},
}
for _, c := range cases {
assert.Equal(t, c.expect, c.input.Severity())
}
}
func TestVulnerabilityList_ApplyWhitelist(t *testing.T) {
cases := []struct {
vl VulnerabilityList
wl models.CVEWhitelist
expectFiltered VulnerabilityList
expectSev models.Severity
}{
{
vl: vulnList2,
wl: whiteList1,
expectFiltered: VulnerabilityList{},
expectSev: models.SevHigh,
},
{
vl: vulnList2,
wl: whiteList2,
expectFiltered: VulnerabilityList{
{
ID: "CVE-2018-6485",
Severity: models.SevHigh,
Pkg: "glibc",
Version: "2.24-11+deb9u4",
},
},
expectSev: models.SevLow,
},
{
vl: vulnList1,
wl: whiteList3,
expectFiltered: VulnerabilityList{},
expectSev: models.SevNone,
},
{
vl: vulnList2,
wl: whiteList3,
expectFiltered: VulnerabilityList{
{ID: "CVE-2018-10754",
Severity: models.SevLow,
Pkg: "ncurses",
Version: "6.0+20161126-1+deb9u2",
},
{
ID: "CVE-2018-6485",
Severity: models.SevHigh,
Pkg: "glibc",
Version: "2.24-11+deb9u4",
},
},
expectSev: models.SevNone,
},
}
for _, c := range cases {
filtered := c.vl.ApplyWhitelist(c.wl)
assert.Equal(t, c.expectFiltered, filtered)
assert.Equal(t, c.vl.Severity(), c.expectSev)
}
}
func TestVulnListByDigest(t *testing.T) {
_, err := VulnListByDigest("notexist")
assert.NotNil(t, err)
}
func TestVulnListFromClairResult(t *testing.T) {
l := VulnListFromClairResult(nil)
assert.Equal(t, VulnerabilityList{}, l)
lv := &models.ClairLayerEnvelope{
Layer: nil,
Error: nil,
}
l2 := VulnListFromClairResult(lv)
assert.Equal(t, VulnerabilityList{}, l2)
}