Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-23 10:45:45 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #2777 from reasonerjt/fetch-timestamp-from-clairdb

Browse Source 
provide default timestamp for all distros in system info api
This commit is contained in:
Daniel Jiang 2017-07-14 15:32:27 +08:00 committed by GitHub
commit 22ba87338e
3 changed files with 127 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/common/models/clair.go
View File

@ -122,3 +122,15 @@ type ClairNamespaceTimestamp struct {
Namespace string `json:"namespace"`
Timestamp int64 `json:"last_update"`
}
//ClairNamespace ...
type ClairNamespace struct {
Name string `json:"Name,omitempty"`
VersionFormat string `json:"VersionFormat,omitempty"`
}
//ClairNamespaceEnvelope ...
type ClairNamespaceEnvelope struct {
Namespaces *[]ClairNamespace `json:"Namespaces,omitempty"`
Error *ClairError `json:"Error,omitempty"`
}

93
src/common/utils/clair/client.go
View File

@ -47,6 +47,22 @@ func NewClient(endpoint string, logger *log.Logger) *Client {
}
}
func (c *Client) send(req *http.Request, expectedStatus int) ([]byte, error) {
resp, err := c.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != expectedStatus {
return nil, fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
}
return b, nil
}
// ScanLayer calls Clair's API to scan a layer.
func (c *Client) ScanLayer(l models.ClairLayer) error {
layer := models.ClairLayerEnvelope{
@ -57,47 +73,28 @@ func (c *Client) ScanLayer(l models.ClairLayer) error {
if err != nil {
return err
}
req, err := http.NewRequest("POST", c.endpoint+"/v1/layers", bytes.NewReader(data))
req, err := http.NewRequest(http.MethodPost, c.endpoint+"/v1/layers", bytes.NewReader(data))
if err != nil {
return err
}
req.Header.Set(http.CanonicalHeaderKey("Content-Type"), "application/json")
resp, err := c.client.Do(req)
_, err = c.send(req, http.StatusCreated)
if err != nil {
return err
log.Errorf("Unexpected error: %v", err)
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
c.logger.Infof("response code: %d", resp.StatusCode)
if resp.StatusCode != http.StatusCreated {
c.logger.Warningf("Unexpected status code: %d", resp.StatusCode)
return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
}
c.logger.Infof("Returning.")
return nil
return err
}
// GetResult calls Clair's API to get layers with detailed vulnerability list
func (c *Client) GetResult(layerName string) (*models.ClairLayerEnvelope, error) {
req, err := http.NewRequest("GET", c.endpoint+"/v1/layers/"+layerName+"?features&vulnerabilities", nil)
req, err := http.NewRequest(http.MethodGet, c.endpoint+"/v1/layers/"+layerName+"?features&vulnerabilities", nil)
if err != nil {
return nil, err
}
resp, err := c.client.Do(req)
b, err := c.send(req, http.StatusOK)
if err != nil {
return nil, err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
}
var res models.ClairLayerEnvelope
err = json.Unmarshal(b, &res)
if err != nil {
@ -108,22 +105,14 @@ func (c *Client) GetResult(layerName string) (*models.ClairLayerEnvelope, error)
// GetNotification calls Clair's API to get details of notification
func (c *Client) GetNotification(id string) (*models.ClairNotification, error) {
req, err := http.NewRequest("GET", c.endpoint+"/v1/notifications/"+id+"?limit=2", nil)
req, err := http.NewRequest(http.MethodGet, c.endpoint+"/v1/notifications/"+id+"?limit=2", nil)
if err != nil {
return nil, err
}
resp, err := c.client.Do(req)
b, err := c.send(req, http.StatusOK)
if err != nil {
return nil, err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
}
var ne models.ClairNotificationEnvelope
err = json.Unmarshal(b, &ne)
if err != nil {
@ -138,22 +127,36 @@ func (c *Client) GetNotification(id string) (*models.ClairNotification, error) {
// DeleteNotification deletes a notification record from Clair
func (c *Client) DeleteNotification(id string) error {
req, err := http.NewRequest("DELETE", c.endpoint+"/v1/notifications/"+id, nil)
req, err := http.NewRequest(http.MethodDelete, c.endpoint+"/v1/notifications/"+id, nil)
if err != nil {
return err
}
resp, err := c.client.Do(req)
_, err = c.send(req, http.StatusOK)
if err != nil {
return err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
}
log.Debugf("Deleted notification %s from Clair.", id)
return nil
}
// ListNamespaces list the namespaces in Clair
func (c *Client) ListNamespaces() ([]string, error) {
req, err := http.NewRequest(http.MethodGet, c.endpoint+"/v1/namespaces", nil)
if err != nil {
return nil, err
}
b, err := c.send(req, http.StatusOK)
if err != nil {
return nil, err
}
var nse models.ClairNamespaceEnvelope
err = json.Unmarshal(b, &nse)
if err != nil {
return nil, err
}
res := []string{}
for _, ns := range *nse.Namespaces {
res = append(res, ns.Name)
}
return res, nil
}

85
src/ui/api/systeminfo.go
View File

@ -19,12 +19,13 @@ import (
"net/http"
"os"
"strings"
"time"
"sync"
"github.com/vmware/harbor/src/common"
"github.com/vmware/harbor/src/common/dao"
clairdao "github.com/vmware/harbor/src/common/dao/clair"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/clair"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/ui/config"
)
@ -48,6 +49,40 @@ type Storage struct {
Free uint64 `json:"free"`
}
//namespaces stores all name spaces on Clair, it should be initialised only once.
type clairNamespaces struct {
*sync.RWMutex
l []string
clair *clair.Client
}
func (n clairNamespaces) get() ([]string, error) {
n.Lock()
defer n.Unlock()
if len(n.l) == 0 {
m := make(map[string]struct{})
if n.clair == nil {
n.clair = clair.NewClient(config.ClairEndpoint(), nil)
}
list, err := n.clair.ListNamespaces()
if err != nil {
return n.l, err
}
for _, n := range list {
ns := strings.Split(n, ":")[0]
m[ns] = struct{}{}
}
for k := range m {
n.l = append(n.l, k)
}
}
return n.l, nil
}
var (
namespaces clairNamespaces
)
//GeneralInfo wraps common systeminfo for anonymous request
type GeneralInfo struct {
WithNotary bool `json:"with_notary"`
@ -166,25 +201,39 @@ func getClairVulnStatus() *models.ClairVulnerabilityStatus {
res.OverallUTC = last
log.Debugf("Clair vuln DB last update: %d", last)
}
l, err := dao.ListClairVulnTimestamps()
if err != nil {
log.Errorf("Failed to list Clair vulnerability timestamps, error:%v", err)
return res
}
m := make(map[string]time.Time)
for _, e := range l {
ns := strings.Split(e.Namespace, ":")
if ts, ok := m[ns[0]]; !ok || ts.Before(e.LastUpdate) {
m[ns[0]] = e.LastUpdate
}
}
details := []models.ClairNamespaceTimestamp{}
for k, v := range m {
e := models.ClairNamespaceTimestamp{
Namespace: k,
Timestamp: v.UTC().Unix(),
if res.OverallUTC > 0 {
l, err := dao.ListClairVulnTimestamps()
if err != nil {
log.Errorf("Failed to list Clair vulnerability timestamps, error:%v", err)
return res
}
m := make(map[string]int64)
for _, e := range l {
ns := strings.Split(e.Namespace, ":")
//only returns the latest time of one distro, i.e. unbuntu:14.04 and ubuntu:15.4 shares one timestamp
el := e.LastUpdate.UTC().Unix()
if ts, ok := m[ns[0]]; !ok || ts < el {
m[ns[0]] = el
}
}
list, err := namespaces.get()
if err != nil {
log.Errorf("Failed to get namespace list from Clair, error: %v", err)
}
//For namespaces not reported by notifier, the timestamp will be the overall db timestamp.
for _, n := range list {
if _, ok := m[n]; !ok {
m[n] = res.OverallUTC
}
}
for k, v := range m {
e := models.ClairNamespaceTimestamp{
Namespace: k,
Timestamp: v,
}
details = append(details, e)
}
details = append(details, e)
}
res.Details = details
return res