Merge pull request #2726 from reasonerjt/clair-notification

clair notification handler enhancements

src/common/const.go

@@ -66,4 +66,6 @@ const (
 	WithNotary                 = "with_notary"
 	WithClair                  = "with_clair"
 	ScanAllPolicy              = "scan_all_policy"
+
+	DefaultClairEndpoint = "http://clair:6060"
 )

src/common/dao/dao_test.go

@@ -1763,3 +1763,14 @@ func TestVulnTimestamp(t *testing.T) {
 		t.Errorf("Delta should be larger than 2 seconds! old: %v, lastupdate: %v", old, res[0].LastUpdate)
 	}
 }
+
+func TestListScanOverviews(t *testing.T) {
+	assert := assert.New(t)
+	err := ClearTable(models.ScanOverviewTable)
+	assert.Nil(err)
+	l, err := ListImgScanOverviews()
+	assert.Nil(err)
+	assert.Equal(0, len(l))
+	err = ClearTable(models.ScanOverviewTable)
+	assert.Nil(err)
+}

src/common/dao/scan_job.go

@@ -146,3 +146,11 @@ func UpdateImgScanOverview(digest, detailsKey string, sev models.Severity, compO
 	}
 	return nil
 }
+
+// ListImgScanOverviews list all records in table img_scan_overview, it is called in notificaiton handler when it needs to refresh the severity of all images.
+func ListImgScanOverviews() ([]*models.ImgScanOverview, error) {
+	var res []*models.ImgScanOverview
+	o := GetOrmer()
+	_, err := o.QueryTable(models.ScanOverviewTable).All(&res)
+	return res, err
+}

src/common/models/clair.go

@@ -107,3 +107,17 @@ type ClairOrderedLayerName struct {
 	Index     int    `json:"Index"`
 	LayerName string `json:"LayerName"`
 }
+
+//ClairVulnerabilityStatus reflects the readiness and freshness of vulnerability data in Clair,
+//which will be returned in response of systeminfo API.
+type ClairVulnerabilityStatus struct {
+	Overall *time.Time                `json:"overall_last_update,omitempty"`
+	Details []ClairNamespaceTimestamp `json:"details,omitempty"`
+}
+
+//ClairNamespaceTimestamp is a record to store the clairname space and the timestamp,
+//in practice different namespace in Clair maybe merged into one, e.g. ubuntu:14.04 and ubuntu:16.4 maybe merged into ubuntu and put into response.
+type ClairNamespaceTimestamp struct {
+	Namespace string    `json:"namespace"`
+	Timestamp time.Time `json:"last_update"`
+}

src/common/utils/clair/client.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"strings"
 	//	"path"
 
 	"github.com/vmware/harbor/src/common/models"
@@ -40,7 +41,7 @@ func NewClient(endpoint string, logger *log.Logger) *Client {
 		logger = log.DefaultLogger()
 	}
 	return &Client{
-		endpoint: endpoint,
+		endpoint: strings.TrimSuffix(endpoint, "/"),
 		logger:   logger,
 		client:   &http.Client{},
 	}

src/common/utils/clair/utils.go

@@ -15,10 +15,17 @@
 package clair
 
 import (
+	"github.com/vmware/harbor/src/common"
+	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils/log"
+
+	"fmt"
 	"strings"
 )
 
+//var client = NewClient()
+
 // ParseClairSev parse the severity of clair to Harbor's Severity type if the string is not recognized the value will be set to unknown.
 func ParseClairSev(clairSev string) models.Severity {
 	sev := strings.ToLower(clairSev)
@@ -35,3 +42,54 @@ func ParseClairSev(clairSev string) models.Severity {
 		return models.SevUnknown
 	}
 }
+
+// UpdateScanOverview qeuries the vulnerability based on the layerName and update the record in img_scan_overview table based on digest.
+func UpdateScanOverview(digest, layerName string, l ...*log.Logger) error {
+	var logger *log.Logger
+	if len(l) > 1 {
+		return fmt.Errorf("More than one logger specified")
+	} else if len(l) == 1 {
+		logger = l[0]
+	} else {
+		logger = log.DefaultLogger()
+	}
+	client := NewClient(common.DefaultClairEndpoint, logger)
+	res, err := client.GetResult(layerName)
+	if err != nil {
+		logger.Errorf("Failed to get result from Clair, error: %v", err)
+		return err
+	}
+	vulnMap := make(map[models.Severity]int)
+	features := res.Layer.Features
+	totalComponents := len(features)
+	logger.Infof("total features: %d", totalComponents)
+	var temp models.Severity
+	for _, f := range features {
+		sev := models.SevNone
+		for _, v := range f.Vulnerabilities {
+			temp = ParseClairSev(v.Severity)
+			if temp > sev {
+				sev = temp
+			}
+		}
+		logger.Infof("Feature: %s, Severity: %d", f.Name, sev)
+		vulnMap[sev]++
+	}
+	overallSev := models.SevNone
+	compSummary := []*models.ComponentsOverviewEntry{}
+	for k, v := range vulnMap {
+		if k > overallSev {
+			overallSev = k
+		}
+		entry := &models.ComponentsOverviewEntry{
+			Sev:   int(k),
+			Count: v,
+		}
+		compSummary = append(compSummary, entry)
+	}
+	compOverview := &models.ComponentsOverview{
+		Total:   totalComponents,
+		Summary: compSummary,
+	}
+	return dao.UpdateImgScanOverview(digest, layerName, overallSev, compOverview)
+}

src/jobservice/config/config.go

@@ -170,5 +170,5 @@ func InternalTokenServiceEndpoint() string {
 
 // ClairEndpoint returns the end point of clair instance, by default it's the one deployed within Harbor.
 func ClairEndpoint() string {
-	return "http://clair:6060"
+	return common.DefaultClairEndpoint
 }

src/jobservice/scan/handlers.go

@@ -17,7 +17,6 @@ package scan
 import (
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/schema2"
-	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/utils/clair"
 	"github.com/vmware/harbor/src/common/utils/registry/auth"
@@ -135,44 +134,9 @@ func (sh *SummarizeHandler) Enter() (string, error) {
 	logger.Infof("Entered summarize handler")
 	layerName := sh.Context.layers[len(sh.Context.layers)-1].Name
 	logger.Infof("Top layer's name: %s, will use it to get the vulnerability result of image", layerName)
-	res, err := sh.Context.clairClient.GetResult(layerName)
-	if err != nil {
-		logger.Errorf("Failed to get result from Clair, error: %v", err)
-		return "", err
+	if err := clair.UpdateScanOverview(sh.Context.Digest, layerName); err != nil {
+		return "", nil
 	}
-	vulnMap := make(map[models.Severity]int)
-	features := res.Layer.Features
-	totalComponents := len(features)
-	logger.Infof("total features: %d", totalComponents)
-	var temp models.Severity
-	for _, f := range features {
-		sev := models.SevNone
-		for _, v := range f.Vulnerabilities {
-			temp = clair.ParseClairSev(v.Severity)
-			if temp > sev {
-				sev = temp
-			}
-		}
-		logger.Infof("Feature: %s, Severity: %d", f.Name, sev)
-		vulnMap[sev]++
-	}
-	overallSev := models.SevNone
-	compSummary := []*models.ComponentsOverviewEntry{}
-	for k, v := range vulnMap {
-		if k > overallSev {
-			overallSev = k
-		}
-		entry := &models.ComponentsOverviewEntry{
-			Sev:   int(k),
-			Count: v,
-		}
-		compSummary = append(compSummary, entry)
-	}
-	compOverview := &models.ComponentsOverview{
-		Total:   totalComponents,
-		Summary: compSummary,
-	}
-	err = dao.UpdateImgScanOverview(sh.Context.Digest, layerName, overallSev, compOverview)
 	return models.JobFinished, nil
 }
 

src/ui/api/config.go

@@ -293,7 +293,7 @@ func validateCfg(c map[string]interface{}) (bool, error) {
 		scope != common.LDAPScopeBase &&
 		scope != common.LDAPScopeOnelevel &&
 		scope != common.LDAPScopeSubtree {
-		return false, fmt.Errorf("invalid %s, should be %s, %s or %s",
+		return false, fmt.Errorf("invalid %s, should be %d, %d or %d",
 			common.LDAPScope,
 			common.LDAPScopeBase,
 			common.LDAPScopeOnelevel,

src/ui/api/systeminfo.go

@@ -19,8 +19,11 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/vmware/harbor/src/common"
+	"github.com/vmware/harbor/src/common/dao"
+	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/ui/config"
 )
@@ -46,16 +49,17 @@ type Storage struct {
 
 //GeneralInfo wraps common systeminfo for anonymous request
 type GeneralInfo struct {
-	WithNotary              bool   `json:"with_notary"`
-	WithClair               bool   `json:"with_clair"`
-	WithAdmiral             bool   `json:"with_admiral"`
-	AdmiralEndpoint         string `json:"admiral_endpoint"`
-	AuthMode                string `json:"auth_mode"`
-	RegistryURL             string `json:"registry_url"`
-	ProjectCreationRestrict string `json:"project_creation_restriction"`
-	SelfRegistration        bool   `json:"self_registration"`
-	HasCARoot               bool   `json:"has_ca_root"`
-	HarborVersion           string `json:"harbor_version"`
+	WithNotary              bool                             `json:"with_notary"`
+	WithClair               bool                             `json:"with_clair"`
+	WithAdmiral             bool                             `json:"with_admiral"`
+	AdmiralEndpoint         string                           `json:"admiral_endpoint"`
+	AuthMode                string                           `json:"auth_mode"`
+	RegistryURL             string                           `json:"registry_url"`
+	ProjectCreationRestrict string                           `json:"project_creation_restriction"`
+	SelfRegistration        bool                             `json:"self_registration"`
+	HasCARoot               bool                             `json:"has_ca_root"`
+	HarborVersion           string                           `json:"harbor_version"`
+	ClairVulnStatus         *models.ClairVulnerabilityStatus `json:"clair_vulnerability_status,omitempty"`
 }
 
 // validate for validating user if an admin.
@@ -134,11 +138,14 @@ func (sia *SystemInfoAPI) GetGeneralInfo() {
 		HasCARoot:               caStatErr == nil,
 		HarborVersion:           harborVersion,
 	}
+	if info.WithClair {
+		info.ClairVulnStatus = getClairVulnStatus()
+	}
 	sia.Data["json"] = info
 	sia.ServeJSON()
 }
 
-// GetVersion gets harbor version.
+// getVersion gets harbor version.
 func (sia *SystemInfoAPI) getVersion() string {
 	version, err := ioutil.ReadFile(harborVersionFile)
 	if err != nil {
@@ -147,3 +154,34 @@ func (sia *SystemInfoAPI) getVersion() string {
 	}
 	return string(version[:])
 }
+
+func getClairVulnStatus() *models.ClairVulnerabilityStatus {
+	res := &models.ClairVulnerabilityStatus{}
+	l, err := dao.ListClairVulnTimestamps()
+	if err != nil {
+		log.Errorf("Failed to list Clair vulnerability timestamps, error:%v", err)
+		return nil
+	}
+	m := make(map[string]time.Time)
+	var t time.Time
+	for _, e := range l {
+		if e.LastUpdate.After(t) {
+			t = e.LastUpdate
+		}
+		ns := strings.Split(e.Namespace, ":")
+		if ts, ok := m[ns[0]]; !ok || ts.Before(e.LastUpdate) {
+			m[ns[0]] = e.LastUpdate
+		}
+	}
+	res.Overall = &t
+	details := []models.ClairNamespaceTimestamp{}
+	for k, v := range m {
+		e := models.ClairNamespaceTimestamp{
+			Namespace: k,
+			Timestamp: v,
+		}
+		details = append(details, e)
+	}
+	res.Details = details
+	return res
+}

src/ui/config/config.go

@@ -355,7 +355,7 @@ func WithClair() bool {
 
 // ClairEndpoint returns the end point of clair instance, by default it's the one deployed within Harbor.
 func ClairEndpoint() string {
-	return "http://clair:6060"
+	return common.DefaultClairEndpoint
 }
 
 // AdmiralEndpoint returns the URL of admiral, if Harbor is not deployed with admiral it should return an empty string.

src/ui/service/notifications/clair/handler.go

@@ -96,7 +96,18 @@ func (h *Handler) Handle() {
 	if rescanTimer.needReschedule() {
 		go func() {
 			<-time.After(rescanInterval)
-			log.Debugf("TODO: rescan or resfresh scan_overview!")
+			l, err := dao.ListImgScanOverviews()
+			if err != nil {
+				log.Errorf("Failed to list scan overview records, error: %v", err)
+				return
+			}
+			for _, e := range l {
+				if err := clair.UpdateScanOverview(e.Digest, e.DetailsKey); err != nil {
+					log.Errorf("Failed to refresh scan overview for image: %s", e.Digest)
+				} else {
+					log.Debugf("Refreshed scan overview for record with digest: %s", e.Digest)
+				}
+			}
 		}()
 	} else {
 		log.Debugf("There is a rescan scheduled already, skip.")