From 72b3a020e368a5d14519a1aa08cb2f1a77f76517 Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Thu, 13 Jul 2017 18:48:05 +0800 Subject: [PATCH] provide default timestamp for all distros in system info api --- src/common/models/clair.go | 12 +++ src/common/utils/clair/client.go | 121 ++++++++++++++++--------------- src/ui/api/systeminfo.go | 85 +++++++++++++++++----- 3 files changed, 141 insertions(+), 77 deletions(-) diff --git a/src/common/models/clair.go b/src/common/models/clair.go index 94f88bdc3..27f9330e4 100644 --- a/src/common/models/clair.go +++ b/src/common/models/clair.go @@ -122,3 +122,15 @@ type ClairNamespaceTimestamp struct { Namespace string `json:"namespace"` Timestamp int64 `json:"last_update"` } + +//ClairNamespace ... +type ClairNamespace struct { + Name string `json:"Name,omitempty"` + VersionFormat string `json:"VersionFormat,omitempty"` +} + +//ClairNamespaceEnvelope ... +type ClairNamespaceEnvelope struct { + Namespaces *[]ClairNamespace `json:"Namespaces,omitempty"` + Error *ClairError `json:"Error,omitempty"` +} diff --git a/src/common/utils/clair/client.go b/src/common/utils/clair/client.go index e0240053c..de604239b 100644 --- a/src/common/utils/clair/client.go +++ b/src/common/utils/clair/client.go @@ -47,45 +47,7 @@ func NewClient(endpoint string, logger *log.Logger) *Client { } } -// ScanLayer calls Clair's API to scan a layer. -func (c *Client) ScanLayer(l models.ClairLayer) error { - layer := models.ClairLayerEnvelope{ - Layer: &l, - Error: nil, - } - data, err := json.Marshal(layer) - if err != nil { - return err - } - req, err := http.NewRequest("POST", c.endpoint+"/v1/layers", bytes.NewReader(data)) - if err != nil { - return err - } - req.Header.Set(http.CanonicalHeaderKey("Content-Type"), "application/json") - resp, err := c.client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - c.logger.Infof("response code: %d", resp.StatusCode) - if resp.StatusCode != http.StatusCreated { - c.logger.Warningf("Unexpected status code: %d", resp.StatusCode) - return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b)) - } - c.logger.Infof("Returning.") - return nil -} - -// GetResult calls Clair's API to get layers with detailed vulnerability list -func (c *Client) GetResult(layerName string) (*models.ClairLayerEnvelope, error) { - req, err := http.NewRequest("GET", c.endpoint+"/v1/layers/"+layerName+"?features&vulnerabilities", nil) - if err != nil { - return nil, err - } +func (c *Client) send(req *http.Request, expectedStatus int) ([]byte, error) { resp, err := c.client.Do(req) if err != nil { return nil, err @@ -98,6 +60,41 @@ func (c *Client) GetResult(layerName string) (*models.ClairLayerEnvelope, error) if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b)) } + return b, nil +} + +// ScanLayer calls Clair's API to scan a layer. +func (c *Client) ScanLayer(l models.ClairLayer) error { + layer := models.ClairLayerEnvelope{ + Layer: &l, + Error: nil, + } + data, err := json.Marshal(layer) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPost, c.endpoint+"/v1/layers", bytes.NewReader(data)) + if err != nil { + return err + } + req.Header.Set(http.CanonicalHeaderKey("Content-Type"), "application/json") + _, err = c.send(req, http.StatusCreated) + if err != nil { + log.Errorf("Unexpected error: %v", err) + } + return err +} + +// GetResult calls Clair's API to get layers with detailed vulnerability list +func (c *Client) GetResult(layerName string) (*models.ClairLayerEnvelope, error) { + req, err := http.NewRequest(http.MethodGet, c.endpoint+"/v1/layers/"+layerName+"?features&vulnerabilities", nil) + if err != nil { + return nil, err + } + b, err := c.send(req, http.StatusOK) + if err != nil { + return nil, err + } var res models.ClairLayerEnvelope err = json.Unmarshal(b, &res) if err != nil { @@ -108,22 +105,14 @@ func (c *Client) GetResult(layerName string) (*models.ClairLayerEnvelope, error) // GetNotification calls Clair's API to get details of notification func (c *Client) GetNotification(id string) (*models.ClairNotification, error) { - req, err := http.NewRequest("GET", c.endpoint+"/v1/notifications/"+id+"?limit=2", nil) + req, err := http.NewRequest(http.MethodGet, c.endpoint+"/v1/notifications/"+id+"?limit=2", nil) if err != nil { return nil, err } - resp, err := c.client.Do(req) + b, err := c.send(req, http.StatusOK) if err != nil { return nil, err } - defer resp.Body.Close() - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b)) - } var ne models.ClairNotificationEnvelope err = json.Unmarshal(b, &ne) if err != nil { @@ -138,22 +127,36 @@ func (c *Client) GetNotification(id string) (*models.ClairNotification, error) { // DeleteNotification deletes a notification record from Clair func (c *Client) DeleteNotification(id string) error { - req, err := http.NewRequest("DELETE", c.endpoint+"/v1/notifications/"+id, nil) + req, err := http.NewRequest(http.MethodDelete, c.endpoint+"/v1/notifications/"+id, nil) if err != nil { return err } - resp, err := c.client.Do(req) + _, err = c.send(req, http.StatusOK) if err != nil { return err } - defer resp.Body.Close() - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b)) - } log.Debugf("Deleted notification %s from Clair.", id) return nil } + +// ListNamespaces list the namespaces in Clair +func (c *Client) ListNamespaces() ([]string, error) { + req, err := http.NewRequest(http.MethodGet, c.endpoint+"/v1/namespaces", nil) + if err != nil { + return nil, err + } + b, err := c.send(req, http.StatusOK) + if err != nil { + return nil, err + } + var nse models.ClairNamespaceEnvelope + err = json.Unmarshal(b, &nse) + if err != nil { + return nil, err + } + res := []string{} + for _, ns := range *nse.Namespaces { + res = append(res, ns.Name) + } + return res, nil +} diff --git a/src/ui/api/systeminfo.go b/src/ui/api/systeminfo.go index 1ea2257ca..eadc7ed87 100644 --- a/src/ui/api/systeminfo.go +++ b/src/ui/api/systeminfo.go @@ -19,12 +19,13 @@ import ( "net/http" "os" "strings" - "time" + "sync" "github.com/vmware/harbor/src/common" "github.com/vmware/harbor/src/common/dao" clairdao "github.com/vmware/harbor/src/common/dao/clair" "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/clair" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/config" ) @@ -48,6 +49,40 @@ type Storage struct { Free uint64 `json:"free"` } +//namespaces stores all name spaces on Clair, it should be initialised only once. +type clairNamespaces struct { + *sync.RWMutex + l []string + clair *clair.Client +} + +func (n clairNamespaces) get() ([]string, error) { + n.Lock() + defer n.Unlock() + if len(n.l) == 0 { + m := make(map[string]struct{}) + if n.clair == nil { + n.clair = clair.NewClient(config.ClairEndpoint(), nil) + } + list, err := n.clair.ListNamespaces() + if err != nil { + return n.l, err + } + for _, n := range list { + ns := strings.Split(n, ":")[0] + m[ns] = struct{}{} + } + for k := range m { + n.l = append(n.l, k) + } + } + return n.l, nil +} + +var ( + namespaces clairNamespaces +) + //GeneralInfo wraps common systeminfo for anonymous request type GeneralInfo struct { WithNotary bool `json:"with_notary"` @@ -166,25 +201,39 @@ func getClairVulnStatus() *models.ClairVulnerabilityStatus { res.OverallUTC = last log.Debugf("Clair vuln DB last update: %d", last) } - l, err := dao.ListClairVulnTimestamps() - if err != nil { - log.Errorf("Failed to list Clair vulnerability timestamps, error:%v", err) - return res - } - m := make(map[string]time.Time) - for _, e := range l { - ns := strings.Split(e.Namespace, ":") - if ts, ok := m[ns[0]]; !ok || ts.Before(e.LastUpdate) { - m[ns[0]] = e.LastUpdate - } - } details := []models.ClairNamespaceTimestamp{} - for k, v := range m { - e := models.ClairNamespaceTimestamp{ - Namespace: k, - Timestamp: v.UTC().Unix(), + if res.OverallUTC > 0 { + l, err := dao.ListClairVulnTimestamps() + if err != nil { + log.Errorf("Failed to list Clair vulnerability timestamps, error:%v", err) + return res + } + m := make(map[string]int64) + for _, e := range l { + ns := strings.Split(e.Namespace, ":") + //only returns the latest time of one distro, i.e. unbuntu:14.04 and ubuntu:15.4 shares one timestamp + el := e.LastUpdate.UTC().Unix() + if ts, ok := m[ns[0]]; !ok || ts < el { + m[ns[0]] = el + } + } + list, err := namespaces.get() + if err != nil { + log.Errorf("Failed to get namespace list from Clair, error: %v", err) + } + //For namespaces not reported by notifier, the timestamp will be the overall db timestamp. + for _, n := range list { + if _, ok := m[n]; !ok { + m[n] = res.OverallUTC + } + } + for k, v := range m { + e := models.ClairNamespaceTimestamp{ + Namespace: k, + Timestamp: v, + } + details = append(details, e) } - details = append(details, e) } res.Details = details return res