From ca805759d945cf9fecfe15806d82992a124cab08 Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Fri, 7 Jul 2017 17:45:08 +0800 Subject: [PATCH] update scan overview in notification handler, and return clair vuln timestamp in system info --- src/common/const.go | 2 + src/common/dao/dao_test.go | 11 ++++ src/common/dao/scan_job.go | 8 +++ src/common/models/clair.go | 14 +++++ src/common/utils/clair/client.go | 3 +- src/common/utils/clair/utils.go | 58 +++++++++++++++++++ src/jobservice/config/config.go | 2 +- src/jobservice/scan/handlers.go | 40 +------------ src/ui/api/config.go | 2 +- src/ui/api/systeminfo.go | 58 +++++++++++++++---- src/ui/config/config.go | 2 +- src/ui/service/notifications/clair/handler.go | 13 ++++- 12 files changed, 159 insertions(+), 54 deletions(-) diff --git a/src/common/const.go b/src/common/const.go index e2bd9c60d5..51215cb305 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -66,4 +66,6 @@ const ( WithNotary = "with_notary" WithClair = "with_clair" ScanAllPolicy = "scan_all_policy" + + DefaultClairEndpoint = "http://clair:6060" ) diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index fd04ae814e..8d8c8375ab 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -1763,3 +1763,14 @@ func TestVulnTimestamp(t *testing.T) { t.Errorf("Delta should be larger than 2 seconds! old: %v, lastupdate: %v", old, res[0].LastUpdate) } } + +func TestListScanOverviews(t *testing.T) { + assert := assert.New(t) + err := ClearTable(models.ScanOverviewTable) + assert.Nil(err) + l, err := ListImgScanOverviews() + assert.Nil(err) + assert.Equal(0, len(l)) + err = ClearTable(models.ScanOverviewTable) + assert.Nil(err) +} diff --git a/src/common/dao/scan_job.go b/src/common/dao/scan_job.go index 51f9ffb682..216203313f 100644 --- a/src/common/dao/scan_job.go +++ b/src/common/dao/scan_job.go @@ -146,3 +146,11 @@ func UpdateImgScanOverview(digest, detailsKey string, sev models.Severity, compO } return nil } + +// ListImgScanOverviews list all records in table img_scan_overview, it is called in notificaiton handler when it needs to refresh the severity of all images. +func ListImgScanOverviews() ([]*models.ImgScanOverview, error) { + var res []*models.ImgScanOverview + o := GetOrmer() + _, err := o.QueryTable(models.ScanOverviewTable).All(&res) + return res, err +} diff --git a/src/common/models/clair.go b/src/common/models/clair.go index da433cf86b..e745f6db82 100644 --- a/src/common/models/clair.go +++ b/src/common/models/clair.go @@ -107,3 +107,17 @@ type ClairOrderedLayerName struct { Index int `json:"Index"` LayerName string `json:"LayerName"` } + +//ClairVulnerabilityStatus reflects the readiness and freshness of vulnerability data in Clair, +//which will be returned in response of systeminfo API. +type ClairVulnerabilityStatus struct { + Overall *time.Time `json:"overall_last_update,omitempty"` + Details []ClairNamespaceTimestamp `json:"details,omitempty"` +} + +//ClairNamespaceTimestamp is a record to store the clairname space and the timestamp, +//in practice different namespace in Clair maybe merged into one, e.g. ubuntu:14.04 and ubuntu:16.4 maybe merged into ubuntu and put into response. +type ClairNamespaceTimestamp struct { + Namespace string `json:"namespace"` + Timestamp time.Time `json:"last_update"` +} diff --git a/src/common/utils/clair/client.go b/src/common/utils/clair/client.go index 40bbef1d0c..e0240053c9 100644 --- a/src/common/utils/clair/client.go +++ b/src/common/utils/clair/client.go @@ -20,6 +20,7 @@ import ( "fmt" "io/ioutil" "net/http" + "strings" // "path" "github.com/vmware/harbor/src/common/models" @@ -40,7 +41,7 @@ func NewClient(endpoint string, logger *log.Logger) *Client { logger = log.DefaultLogger() } return &Client{ - endpoint: endpoint, + endpoint: strings.TrimSuffix(endpoint, "/"), logger: logger, client: &http.Client{}, } diff --git a/src/common/utils/clair/utils.go b/src/common/utils/clair/utils.go index fd9a1310bc..f4f3fdfc1b 100644 --- a/src/common/utils/clair/utils.go +++ b/src/common/utils/clair/utils.go @@ -15,10 +15,17 @@ package clair import ( + "github.com/vmware/harbor/src/common" + "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" + + "fmt" "strings" ) +//var client = NewClient() + // ParseClairSev parse the severity of clair to Harbor's Severity type if the string is not recognized the value will be set to unknown. func ParseClairSev(clairSev string) models.Severity { sev := strings.ToLower(clairSev) @@ -35,3 +42,54 @@ func ParseClairSev(clairSev string) models.Severity { return models.SevUnknown } } + +// UpdateScanOverview qeuries the vulnerability based on the layerName and update the record in img_scan_overview table based on digest. +func UpdateScanOverview(digest, layerName string, l ...*log.Logger) error { + var logger *log.Logger + if len(l) > 1 { + return fmt.Errorf("More than one logger specified") + } else if len(l) == 1 { + logger = l[0] + } else { + logger = log.DefaultLogger() + } + client := NewClient(common.DefaultClairEndpoint, logger) + res, err := client.GetResult(layerName) + if err != nil { + logger.Errorf("Failed to get result from Clair, error: %v", err) + return err + } + vulnMap := make(map[models.Severity]int) + features := res.Layer.Features + totalComponents := len(features) + logger.Infof("total features: %d", totalComponents) + var temp models.Severity + for _, f := range features { + sev := models.SevNone + for _, v := range f.Vulnerabilities { + temp = ParseClairSev(v.Severity) + if temp > sev { + sev = temp + } + } + logger.Infof("Feature: %s, Severity: %d", f.Name, sev) + vulnMap[sev]++ + } + overallSev := models.SevNone + compSummary := []*models.ComponentsOverviewEntry{} + for k, v := range vulnMap { + if k > overallSev { + overallSev = k + } + entry := &models.ComponentsOverviewEntry{ + Sev: int(k), + Count: v, + } + compSummary = append(compSummary, entry) + } + compOverview := &models.ComponentsOverview{ + Total: totalComponents, + Summary: compSummary, + } + return dao.UpdateImgScanOverview(digest, layerName, overallSev, compOverview) +} diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index e362d24518..b3c79f6ea5 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -170,5 +170,5 @@ func InternalTokenServiceEndpoint() string { // ClairEndpoint returns the end point of clair instance, by default it's the one deployed within Harbor. func ClairEndpoint() string { - return "http://clair:6060" + return common.DefaultClairEndpoint } diff --git a/src/jobservice/scan/handlers.go b/src/jobservice/scan/handlers.go index 1d4edec822..66f29b2f49 100644 --- a/src/jobservice/scan/handlers.go +++ b/src/jobservice/scan/handlers.go @@ -17,7 +17,6 @@ package scan import ( "github.com/docker/distribution" "github.com/docker/distribution/manifest/schema2" - "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/clair" "github.com/vmware/harbor/src/common/utils/registry/auth" @@ -135,44 +134,9 @@ func (sh *SummarizeHandler) Enter() (string, error) { logger.Infof("Entered summarize handler") layerName := sh.Context.layers[len(sh.Context.layers)-1].Name logger.Infof("Top layer's name: %s, will use it to get the vulnerability result of image", layerName) - res, err := sh.Context.clairClient.GetResult(layerName) - if err != nil { - logger.Errorf("Failed to get result from Clair, error: %v", err) - return "", err + if err := clair.UpdateScanOverview(sh.Context.Digest, layerName); err != nil { + return "", nil } - vulnMap := make(map[models.Severity]int) - features := res.Layer.Features - totalComponents := len(features) - logger.Infof("total features: %d", totalComponents) - var temp models.Severity - for _, f := range features { - sev := models.SevNone - for _, v := range f.Vulnerabilities { - temp = clair.ParseClairSev(v.Severity) - if temp > sev { - sev = temp - } - } - logger.Infof("Feature: %s, Severity: %d", f.Name, sev) - vulnMap[sev]++ - } - overallSev := models.SevNone - compSummary := []*models.ComponentsOverviewEntry{} - for k, v := range vulnMap { - if k > overallSev { - overallSev = k - } - entry := &models.ComponentsOverviewEntry{ - Sev: int(k), - Count: v, - } - compSummary = append(compSummary, entry) - } - compOverview := &models.ComponentsOverview{ - Total: totalComponents, - Summary: compSummary, - } - err = dao.UpdateImgScanOverview(sh.Context.Digest, layerName, overallSev, compOverview) return models.JobFinished, nil } diff --git a/src/ui/api/config.go b/src/ui/api/config.go index a1b6f8b1b9..7b5af6cce8 100644 --- a/src/ui/api/config.go +++ b/src/ui/api/config.go @@ -293,7 +293,7 @@ func validateCfg(c map[string]interface{}) (bool, error) { scope != common.LDAPScopeBase && scope != common.LDAPScopeOnelevel && scope != common.LDAPScopeSubtree { - return false, fmt.Errorf("invalid %s, should be %s, %s or %s", + return false, fmt.Errorf("invalid %s, should be %d, %d or %d", common.LDAPScope, common.LDAPScopeBase, common.LDAPScopeOnelevel, diff --git a/src/ui/api/systeminfo.go b/src/ui/api/systeminfo.go index 0d161ffcb7..a31ddc2098 100644 --- a/src/ui/api/systeminfo.go +++ b/src/ui/api/systeminfo.go @@ -19,8 +19,11 @@ import ( "net/http" "os" "strings" + "time" "github.com/vmware/harbor/src/common" + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/config" ) @@ -46,16 +49,17 @@ type Storage struct { //GeneralInfo wraps common systeminfo for anonymous request type GeneralInfo struct { - WithNotary bool `json:"with_notary"` - WithClair bool `json:"with_clair"` - WithAdmiral bool `json:"with_admiral"` - AdmiralEndpoint string `json:"admiral_endpoint"` - AuthMode string `json:"auth_mode"` - RegistryURL string `json:"registry_url"` - ProjectCreationRestrict string `json:"project_creation_restriction"` - SelfRegistration bool `json:"self_registration"` - HasCARoot bool `json:"has_ca_root"` - HarborVersion string `json:"harbor_version"` + WithNotary bool `json:"with_notary"` + WithClair bool `json:"with_clair"` + WithAdmiral bool `json:"with_admiral"` + AdmiralEndpoint string `json:"admiral_endpoint"` + AuthMode string `json:"auth_mode"` + RegistryURL string `json:"registry_url"` + ProjectCreationRestrict string `json:"project_creation_restriction"` + SelfRegistration bool `json:"self_registration"` + HasCARoot bool `json:"has_ca_root"` + HarborVersion string `json:"harbor_version"` + ClairVulnStatus *models.ClairVulnerabilityStatus `json:"clair_vulnerability_status,omitempty"` } // validate for validating user if an admin. @@ -133,12 +137,13 @@ func (sia *SystemInfoAPI) GetGeneralInfo() { RegistryURL: registryURL, HasCARoot: caStatErr == nil, HarborVersion: harborVersion, + ClairVulnStatus: getClairVulnStatus(), } sia.Data["json"] = info sia.ServeJSON() } -// GetVersion gets harbor version. +// getVersion gets harbor version. func (sia *SystemInfoAPI) getVersion() string { version, err := ioutil.ReadFile(harborVersionFile) if err != nil { @@ -147,3 +152,34 @@ func (sia *SystemInfoAPI) getVersion() string { } return string(version[:]) } + +func getClairVulnStatus() *models.ClairVulnerabilityStatus { + res := &models.ClairVulnerabilityStatus{} + l, err := dao.ListClairVulnTimestamps() + if err != nil { + log.Errorf("Failed to list Clair vulnerability timestamps, error:%v", err) + return nil + } + m := make(map[string]time.Time) + var t time.Time + for _, e := range l { + if e.LastUpdate.After(t) { + t = e.LastUpdate + } + ns := strings.Split(e.Namespace, ":") + if ts, ok := m[ns[0]]; !ok || ts.Before(e.LastUpdate) { + m[ns[0]] = e.LastUpdate + } + } + res.Overall = &t + details := []models.ClairNamespaceTimestamp{} + for k, v := range m { + e := models.ClairNamespaceTimestamp{ + Namespace: k, + Timestamp: v, + } + details = append(details, e) + } + res.Details = details + return res +} diff --git a/src/ui/config/config.go b/src/ui/config/config.go index 2c0287dbc1..e85cd69ff2 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -355,7 +355,7 @@ func WithClair() bool { // ClairEndpoint returns the end point of clair instance, by default it's the one deployed within Harbor. func ClairEndpoint() string { - return "http://clair:6060" + return common.DefaultClairEndpoint } // AdmiralEndpoint returns the URL of admiral, if Harbor is not deployed with admiral it should return an empty string. diff --git a/src/ui/service/notifications/clair/handler.go b/src/ui/service/notifications/clair/handler.go index 850fbd8171..637bd233e1 100644 --- a/src/ui/service/notifications/clair/handler.go +++ b/src/ui/service/notifications/clair/handler.go @@ -96,7 +96,18 @@ func (h *Handler) Handle() { if rescanTimer.needReschedule() { go func() { <-time.After(rescanInterval) - log.Debugf("TODO: rescan or resfresh scan_overview!") + l, err := dao.ListImgScanOverviews() + if err != nil { + log.Errorf("Failed to list scan overview records, error: %v", err) + return + } + for _, e := range l { + if err := clair.UpdateScanOverview(e.Digest, e.DetailsKey); err != nil { + log.Errorf("Failed to refresh scan overview for image: %s", e.Digest) + } else { + log.Debugf("Refreshed scan overview for record with digest: %s", e.Digest) + } + } }() } else { log.Debugf("There is a rescan scheduled already, skip.")