Merge remote-tracking branch 'upstream/master'

Makefile

@@ -92,7 +92,7 @@ REBUILDCLARITYFLAG=false
 NEWCLARITYVERSION=
 
 #clair parameters
-CLAIRVERSION=v2.0.1
+CLAIRVERSION=v2.0.1-photon
 CLAIRFLAG=false
 CLAIRDBVERSION=9.6.3-photon
 
@@ -243,7 +243,7 @@ ifeq ($(NOTARYFLAG), true)
 	DOCKERCOMPOSE_LIST+= -f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSENOTARYFILENAME)
 endif
 ifeq ($(CLAIRFLAG), true)
-	DOCKERSAVE_PARA+= quay.io/coreos/clair:$(CLAIRVERSION) vmware/postgresql:$(CLAIRDBVERSION)
+	DOCKERSAVE_PARA+= vmware/clair:$(CLAIRVERSION) vmware/postgresql:$(CLAIRDBVERSION)
 	PACKAGE_OFFLINE_PARA+= $(HARBORPKG)/$(DOCKERCOMPOSECLAIRFILENAME)
 	PACKAGE_ONLINE_PARA+= $(HARBORPKG)/$(DOCKERCOMPOSECLAIRFILENAME)
 	DOCKERCOMPOSE_LIST+= -f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
@@ -368,7 +368,7 @@ package_offline: compile build modify_sourcefiles modify_composefile
 	fi
 	@if [ "$(CLAIRFLAG)" = "true" ] ; then \
 		echo "pulling claiy and postgres..."; \
-		$(DOCKERPULL) quay.io/coreos/clair:$(CLAIRVERSION); \
+		$(DOCKERPULL) vmware/clair:$(CLAIRVERSION); \
 		$(DOCKERPULL) vmware/postgresql:$(CLAIRDBVERSION); \
 	fi
 

make/common/db/registry.sql

@@ -181,6 +181,7 @@ create table img_scan_job (
  );
 
 create table img_scan_overview (
+ id int NOT NULL AUTO_INCREMENT,
  image_digest varchar(128) NOT NULL,
  scan_job_id int NOT NULL,
  /* 0 indicates none, the higher the number, the more severe the status */
@@ -191,7 +192,8 @@ create table img_scan_overview (
  details_key varchar(128),
  creation_time timestamp default CURRENT_TIMESTAMP,
  update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
- PRIMARY KEY(image_digest)
+ PRIMARY KEY(id),
+ UNIQUE(image_digest)
  );
 
 create table clair_vuln_timestamp (

make/common/db/registry_sqlite.sql

@@ -172,7 +172,8 @@ create table img_scan_job (
  );
 
 create table img_scan_overview (
- image_digest varchar(128) PRIMARY KEY,
+ id INTEGER PRIMARY KEY, 
+ image_digest varchar(128),
  scan_job_id int NOT NULL,
  /* 0 indicates none, the higher the number, the more severe the status */
  severity int NOT NULL default 0,
@@ -181,7 +182,8 @@ create table img_scan_overview (
  /* primary key for querying details, in clair it should be the name of the "top layer" */
  details_key varchar(128),
  creation_time timestamp default CURRENT_TIMESTAMP,
- update_time timestamp default CURRENT_TIMESTAMP 
+ update_time timestamp default CURRENT_TIMESTAMP,
+ UNIQUE(image_digest)
  );
 
 CREATE INDEX policy ON replication_job (policy_id);

make/docker-compose.clair.yml

@@ -35,7 +35,7 @@ services:
     networks:
       - harbor-clair
     container_name: clair
-    image: quay.io/coreos/clair:v2.0.1
+    image: vmware/clair:v2.0.1-photon
     restart: always
     depends_on:
       - postgres

make/photon/clair/Dockerfile

@@ -0,0 +1,13 @@
+FROM library/photon:1.0
+
+RUN tdnf install -y git bzr rpm xz \
+	&& mkdir /clair2.0.1/ 
+
+COPY clair /clair2.0.1/
+
+VOLUME /config
+EXPOSE 6060 6061	
+
+RUN chmod u+x /clair2.0.1/clair
+
+ENTRYPOINT ["/clair2.0.1/clair"]

src/common/api/base.go

@@ -23,6 +23,7 @@ import (
 	"github.com/astaxie/beego/validation"
 	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
+	http_error "github.com/vmware/harbor/src/common/utils/error"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/ui/auth"
 
@@ -80,6 +81,21 @@ func (b *BaseAPI) HandleInternalServerError(text string) {
 	b.RenderError(http.StatusInternalServerError, "")
 }
 
+// ParseAndHandleError : if the err is an instance of utils/error.Error,
+// return the status code and the detail message contained in err, otherwise
+// return 500
+func (b *BaseAPI) ParseAndHandleError(text string, err error) {
+	if err == nil {
+		return
+	}
+	log.Errorf("%s: %v", text, err)
+	if e, ok := err.(*http_error.HTTPError); ok {
+		b.RenderError(e.StatusCode, e.Detail)
+		return
+	}
+	b.RenderError(http.StatusInternalServerError, "")
+}
+
 // Render returns nil as it won't render template
 func (b *BaseAPI) Render() error {
 	return nil

src/common/const.go

@@ -66,4 +66,6 @@ const (
 	WithNotary                 = "with_notary"
 	WithClair                  = "with_clair"
 	ScanAllPolicy              = "scan_all_policy"
+
+	DefaultClairEndpoint = "http://clair:6060"
 )

src/common/dao/dao_test.go

@@ -1763,3 +1763,14 @@ func TestVulnTimestamp(t *testing.T) {
 		t.Errorf("Delta should be larger than 2 seconds! old: %v, lastupdate: %v", old, res[0].LastUpdate)
 	}
 }
+
+func TestListScanOverviews(t *testing.T) {
+	assert := assert.New(t)
+	err := ClearTable(models.ScanOverviewTable)
+	assert.Nil(err)
+	l, err := ListImgScanOverviews()
+	assert.Nil(err)
+	assert.Equal(0, len(l))
+	err = ClearTable(models.ScanOverviewTable)
+	assert.Nil(err)
+}

src/common/dao/scan_job.go

@@ -95,6 +95,7 @@ func SetScanJobForImg(digest string, jobID int64) error {
 	}
 	if !created {
 		rec.JobID = jobID
+		rec.UpdateTime = time.Now()
 		n, err := o.Update(rec, "JobID", "UpdateTime")
 		if n == 0 {
 			return fmt.Errorf("Failed to set scan job for image with digest: %s, error: %v", digest, err)
@@ -105,17 +106,18 @@ func SetScanJobForImg(digest string, jobID int64) error {
 
 // GetImgScanOverview returns the ImgScanOverview based on the digest.
 func GetImgScanOverview(digest string) (*models.ImgScanOverview, error) {
-	o := GetOrmer()
-	rec := &models.ImgScanOverview{
-		Digest: digest,
-	}
-	err := o.Read(rec)
-	if err != nil && err != orm.ErrNoRows {
+	res := []*models.ImgScanOverview{}
+	_, err := scanOverviewQs().Filter("image_digest", digest).All(&res)
+	if err != nil {
 		return nil, err
 	}
-	if err == orm.ErrNoRows {
+	if len(res) == 0 {
 		return nil, nil
 	}
+	if len(res) > 1 {
+		return nil, fmt.Errorf("Found multiple scan_overview entries for digest: %s", digest)
+	}
+	rec := res[0]
 	if len(rec.CompOverviewStr) > 0 {
 		co := &models.ComponentsOverview{}
 		if err := json.Unmarshal([]byte(rec.CompOverviewStr), co); err != nil {
@@ -129,20 +131,38 @@ func GetImgScanOverview(digest string) (*models.ImgScanOverview, error) {
 // UpdateImgScanOverview updates the serverity and components status of a record in img_scan_overview
 func UpdateImgScanOverview(digest, detailsKey string, sev models.Severity, compOverview *models.ComponentsOverview) error {
 	o := GetOrmer()
+	rec, err := GetImgScanOverview(digest)
+	if err != nil {
+		return fmt.Errorf("Failed to getting scan_overview record for update: %v", err)
+	}
+	if rec == nil {
+		return fmt.Errorf("No scan_overview record for digest: %s", digest)
+	}
 	b, err := json.Marshal(compOverview)
 	if err != nil {
 		return err
 	}
-	rec := &models.ImgScanOverview{
-		Digest:          digest,
-		Sev:             int(sev),
-		CompOverviewStr: string(b),
-		DetailsKey:      detailsKey,
-		UpdateTime:      time.Now(),
-	}
+	rec.Sev = int(sev)
+	rec.CompOverviewStr = string(b)
+	rec.DetailsKey = detailsKey
+	rec.UpdateTime = time.Now()
+
 	n, err := o.Update(rec, "Sev", "CompOverviewStr", "DetailsKey", "UpdateTime")
 	if n == 0 || err != nil {
 		return fmt.Errorf("Failed to update scan overview record with digest: %s, error: %v", digest, err)
 	}
 	return nil
 }
+
+// ListImgScanOverviews list all records in table img_scan_overview, it is called in notificaiton handler when it needs to refresh the severity of all images.
+func ListImgScanOverviews() ([]*models.ImgScanOverview, error) {
+	var res []*models.ImgScanOverview
+	o := GetOrmer()
+	_, err := o.QueryTable(models.ScanOverviewTable).All(&res)
+	return res, err
+}
+
+func scanOverviewQs() orm.QuerySeter {
+	o := GetOrmer()
+	return o.QueryTable(models.ScanOverviewTable)
+}

src/common/models/clair.go

@@ -107,3 +107,17 @@ type ClairOrderedLayerName struct {
 	Index     int    `json:"Index"`
 	LayerName string `json:"LayerName"`
 }
+
+//ClairVulnerabilityStatus reflects the readiness and freshness of vulnerability data in Clair,
+//which will be returned in response of systeminfo API.
+type ClairVulnerabilityStatus struct {
+	Overall *time.Time                `json:"overall_last_update,omitempty"`
+	Details []ClairNamespaceTimestamp `json:"details,omitempty"`
+}
+
+//ClairNamespaceTimestamp is a record to store the clairname space and the timestamp,
+//in practice different namespace in Clair maybe merged into one, e.g. ubuntu:14.04 and ubuntu:16.4 maybe merged into ubuntu and put into response.
+type ClairNamespaceTimestamp struct {
+	Namespace string    `json:"namespace"`
+	Timestamp time.Time `json:"last_update"`
+}

src/common/models/scan_job.go

@@ -53,7 +53,8 @@ func (s *ScanJob) TableName() string {
 
 //ImgScanOverview mapped to a record of image scan overview.
 type ImgScanOverview struct {
-	Digest          string              `orm:"pk;column(image_digest)" json:"image_digest"`
+	ID              int64               `orm:"pk;auto;column(id)" json:"-"`
+	Digest          string              `orm:"column(image_digest)" json:"image_digest"`
 	Status          string              `orm:"-" json:"scan_status"`
 	JobID           int64               `orm:"column(scan_job_id)" json:"job_id"`
 	Sev             int                 `orm:"column(severity)" json:"severity"`

src/common/security/admiral/authcontext/authcontext.go

@@ -26,6 +26,7 @@ import (
 	"github.com/vmware/harbor/src/common"
 	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/utils"
+	http_error "github.com/vmware/harbor/src/common/utils/error"
 	"github.com/vmware/harbor/src/common/utils/log"
 )
 
@@ -202,7 +203,10 @@ func send(client *http.Client, req *http.Request) (*AuthContext, error) {
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("unexpected status code: %d %s", resp.StatusCode, string(data))
+		return nil, &http_error.HTTPError{
+			StatusCode: resp.StatusCode,
+			Detail:     string(data),
+		}
 	}
 
 	ctx := &AuthContext{}

src/common/utils/clair/client.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"strings"
 	//	"path"
 
 	"github.com/vmware/harbor/src/common/models"
@@ -40,7 +41,7 @@ func NewClient(endpoint string, logger *log.Logger) *Client {
 		logger = log.DefaultLogger()
 	}
 	return &Client{
-		endpoint: endpoint,
+		endpoint: strings.TrimSuffix(endpoint, "/"),
 		logger:   logger,
 		client:   &http.Client{},
 	}

src/common/utils/clair/utils.go

@@ -15,10 +15,17 @@
 package clair
 
 import (
+	"github.com/vmware/harbor/src/common"
+	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils/log"
+
+	"fmt"
 	"strings"
 )
 
+//var client = NewClient()
+
 // ParseClairSev parse the severity of clair to Harbor's Severity type if the string is not recognized the value will be set to unknown.
 func ParseClairSev(clairSev string) models.Severity {
 	sev := strings.ToLower(clairSev)
@@ -35,3 +42,54 @@ func ParseClairSev(clairSev string) models.Severity {
 		return models.SevUnknown
 	}
 }
+
+// UpdateScanOverview qeuries the vulnerability based on the layerName and update the record in img_scan_overview table based on digest.
+func UpdateScanOverview(digest, layerName string, l ...*log.Logger) error {
+	var logger *log.Logger
+	if len(l) > 1 {
+		return fmt.Errorf("More than one logger specified")
+	} else if len(l) == 1 {
+		logger = l[0]
+	} else {
+		logger = log.DefaultLogger()
+	}
+	client := NewClient(common.DefaultClairEndpoint, logger)
+	res, err := client.GetResult(layerName)
+	if err != nil {
+		logger.Errorf("Failed to get result from Clair, error: %v", err)
+		return err
+	}
+	vulnMap := make(map[models.Severity]int)
+	features := res.Layer.Features
+	totalComponents := len(features)
+	logger.Infof("total features: %d", totalComponents)
+	var temp models.Severity
+	for _, f := range features {
+		sev := models.SevNone
+		for _, v := range f.Vulnerabilities {
+			temp = ParseClairSev(v.Severity)
+			if temp > sev {
+				sev = temp
+			}
+		}
+		logger.Infof("Feature: %s, Severity: %d", f.Name, sev)
+		vulnMap[sev]++
+	}
+	overallSev := models.SevNone
+	compSummary := []*models.ComponentsOverviewEntry{}
+	for k, v := range vulnMap {
+		if k > overallSev {
+			overallSev = k
+		}
+		entry := &models.ComponentsOverviewEntry{
+			Sev:   int(k),
+			Count: v,
+		}
+		compSummary = append(compSummary, entry)
+	}
+	compOverview := &models.ComponentsOverview{
+		Total:   totalComponents,
+		Summary: compSummary,
+	}
+	return dao.UpdateImgScanOverview(digest, layerName, overallSev, compOverview)
+}

src/common/utils/error/error.go

@@ -18,13 +18,13 @@ import (
 	"fmt"
 )
 
-// Error : if response is returned but the status code is not 200, an Error instance will be returned
-type Error struct {
+// HTTPError : if response is returned but the status code is not 200, an Error instance will be returned
+type HTTPError struct {
 	StatusCode int
 	Detail     string
 }
 
 // Error returns the details as string
-func (e *Error) Error() string {
+func (e *HTTPError) Error() string {
 	return fmt.Sprintf("%d %s", e.StatusCode, e.Detail)
 }

src/common/utils/error/error_test.go

@@ -18,7 +18,7 @@ import (
 )
 
 func TestError(t *testing.T) {
-	err := &Error{
+	err := &HTTPError{
 		StatusCode: 404,
 		Detail:     "not found",
 	}

src/common/utils/registry/auth/util.go

@@ -78,7 +78,7 @@ func getToken(client *http.Client, credential Credential, realm, service string,
 		return nil, err
 	}
 	if resp.StatusCode != http.StatusOK {
-		return nil, &registry_error.Error{
+		return nil, &registry_error.HTTPError{
 			StatusCode: resp.StatusCode,
 			Detail:     string(data),
 		}

src/common/utils/registry/registry.go

@@ -126,7 +126,7 @@ func (r *Registry) Catalog() ([]string, error) {
 				suffix = ""
 			}
 		} else {
-			return repos, &registry_error.Error{
+			return repos, &registry_error.HTTPError{
 				StatusCode: resp.StatusCode,
 				Detail:     string(b),
 			}
@@ -157,7 +157,7 @@ func (r *Registry) Ping() error {
 		return err
 	}
 
-	return &registry_error.Error{
+	return &registry_error.HTTPError{
 		StatusCode: resp.StatusCode,
 		Detail:     string(b),
 	}

src/common/utils/registry/repository.go

@@ -72,7 +72,7 @@ func NewRepositoryWithModifiers(name, endpoint string, insecure bool, modifiers
 
 func parseError(err error) error {
 	if urlErr, ok := err.(*url.Error); ok {
-		if regErr, ok := urlErr.Err.(*registry_error.Error); ok {
+		if regErr, ok := urlErr.Err.(*registry_error.HTTPError); ok {
 			return regErr
 		}
 	}
@@ -120,7 +120,7 @@ func (r *Repository) ListTag() ([]string, error) {
 		return tags, nil
 	}
 
-	return tags, &registry_error.Error{
+	return tags, &registry_error.HTTPError{
 		StatusCode: resp.StatusCode,
 		Detail:     string(b),
 	}
@@ -160,7 +160,7 @@ func (r *Repository) ManifestExist(reference string) (digest string, exist bool,
 		return
 	}
 
-	err = &registry_error.Error{
+	err = &registry_error.HTTPError{
 		StatusCode: resp.StatusCode,
 		Detail:     string(b),
 	}
@@ -197,7 +197,7 @@ func (r *Repository) PullManifest(reference string, acceptMediaTypes []string) (
 		return
 	}
 
-	err = &registry_error.Error{
+	err = &registry_error.HTTPError{
 		StatusCode: resp.StatusCode,
 		Detail:     string(b),
 	}
@@ -232,7 +232,7 @@ func (r *Repository) PushManifest(reference, mediaType string, payload []byte) (
 		return
 	}
 
-	err = &registry_error.Error{
+	err = &registry_error.HTTPError{
 		StatusCode: resp.StatusCode,
 		Detail:     string(b),
 	}
@@ -263,7 +263,7 @@ func (r *Repository) DeleteManifest(digest string) error {
 		return err
 	}
 
-	return &registry_error.Error{
+	return &registry_error.HTTPError{
 		StatusCode: resp.StatusCode,
 		Detail:     string(b),
 	}
@@ -277,7 +277,7 @@ func (r *Repository) DeleteTag(tag string) error {
 	}
 
 	if !exist {
-		return &registry_error.Error{
+		return &registry_error.HTTPError{
 			StatusCode: http.StatusNotFound,
 		}
 	}
@@ -312,7 +312,7 @@ func (r *Repository) BlobExist(digest string) (bool, error) {
 		return false, err
 	}
 
-	return false, &registry_error.Error{
+	return false, &registry_error.HTTPError{
 		StatusCode: resp.StatusCode,
 		Detail:     string(b),
 	}
@@ -348,7 +348,7 @@ func (r *Repository) PullBlob(digest string) (size int64, data io.ReadCloser, er
 		return
 	}
 
-	err = &registry_error.Error{
+	err = &registry_error.HTTPError{
 		StatusCode: resp.StatusCode,
 		Detail:     string(b),
 	}
@@ -379,7 +379,7 @@ func (r *Repository) initiateBlobUpload(name string) (location, uploadUUID strin
 		return
 	}
 
-	err = &registry_error.Error{
+	err = &registry_error.HTTPError{
 		StatusCode: resp.StatusCode,
 		Detail:     string(b),
 	}
@@ -409,7 +409,7 @@ func (r *Repository) monolithicBlobUpload(location, digest string, size int64, d
 		return err
 	}
 
-	return &registry_error.Error{
+	return &registry_error.HTTPError{
 		StatusCode: resp.StatusCode,
 		Detail:     string(b),
 	}
@@ -447,7 +447,7 @@ func (r *Repository) DeleteBlob(digest string) error {
 		return err
 	}
 
-	return &registry_error.Error{
+	return &registry_error.HTTPError{
 		StatusCode: resp.StatusCode,
 		Detail:     string(b),
 	}

src/common/utils/registry/repository_test.go

@@ -396,10 +396,10 @@ func TestListTag(t *testing.T) {
 
 func TestParseError(t *testing.T) {
 	err := &url.Error{
-		Err: &registry_error.Error{},
+		Err: &registry_error.HTTPError{},
 	}
 	e := parseError(err)
-	if _, ok := e.(*registry_error.Error); !ok {
+	if _, ok := e.(*registry_error.HTTPError); !ok {
 		t.Errorf("error type does not match registry error")
 	}
 }

src/common/utils/timemarker.go

@@ -0,0 +1,78 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"os"
+	"strconv"
+	"sync"
+	"time"
+)
+
+var (
+	scanAllMarker      *TimeMarker
+	scanOverviewMarker = &TimeMarker{
+		interval: 15 * time.Second,
+	}
+	once sync.Once
+)
+
+//TimeMarker is used to control an action not to be taken frequently within the interval
+type TimeMarker struct {
+	sync.RWMutex
+	next     time.Time
+	interval time.Duration
+}
+
+//Mark tries to mark a future time, which is after the duration of interval from the time it's called.
+//It returns false if there is a mark in fugure, true if the mark is successfully set.
+func (t *TimeMarker) Mark() bool {
+	t.Lock()
+	defer t.Unlock()
+	if time.Now().Before(t.next) {
+		return false
+	}
+	t.next = time.Now().Add(t.interval)
+	return true
+}
+
+//Next returns the time of the next mark.
+func (t *TimeMarker) Next() time.Time {
+	t.RLock()
+	defer t.RUnlock()
+	return t.next
+}
+
+//ScanAllMarker ...
+func ScanAllMarker() *TimeMarker {
+	once.Do(func() {
+		a := os.Getenv("HARBOR_SCAN_ALL_INTERVAL")
+		if m, err := strconv.Atoi(a); err == nil {
+			scanAllMarker = &TimeMarker{
+				interval: time.Duration(m) * time.Minute,
+			}
+		} else {
+			scanAllMarker = &TimeMarker{
+				interval: 30 * time.Minute,
+			}
+		}
+	})
+	return scanAllMarker
+}
+
+//ScanOverviewMarker ...
+func ScanOverviewMarker() *TimeMarker {
+	return scanOverviewMarker
+}

src/common/utils/timemarker_test.go

@@ -0,0 +1,48 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestTimeMarker(t *testing.T) {
+	assert := assert.New(t)
+	m := &TimeMarker{
+		interval: 1 * time.Second,
+	}
+	r1 := m.Mark()
+	assert.True(r1)
+	r2 := m.Mark()
+	assert.False(r2)
+	t.Log("Sleep for 2 seconds...")
+	time.Sleep(2 * time.Second)
+	r3 := m.Mark()
+	assert.True(r3)
+}
+
+func TestScanMarkers(t *testing.T) {
+	assert := assert.New(t)
+	os.Setenv("HARBOR_SCAN_ALL_INTERVAL", "5")
+	sm := ScanAllMarker()
+	d := sm.Next().Sub(time.Now())
+	assert.True(d <= 5*time.Minute)
+	som := ScanOverviewMarker()
+	d = som.Next().Sub(time.Now())
+	assert.True(d <= 15*time.Second)
+}

src/jobservice/config/config.go

@@ -170,5 +170,5 @@ func InternalTokenServiceEndpoint() string {
 
 // ClairEndpoint returns the end point of clair instance, by default it's the one deployed within Harbor.
 func ClairEndpoint() string {
-	return "http://clair:6060"
+	return common.DefaultClairEndpoint
 }

src/jobservice/scan/handlers.go

@@ -17,7 +17,6 @@ package scan
 import (
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/schema2"
-	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/utils/clair"
 	"github.com/vmware/harbor/src/common/utils/registry/auth"
@@ -135,44 +134,9 @@ func (sh *SummarizeHandler) Enter() (string, error) {
 	logger.Infof("Entered summarize handler")
 	layerName := sh.Context.layers[len(sh.Context.layers)-1].Name
 	logger.Infof("Top layer's name: %s, will use it to get the vulnerability result of image", layerName)
-	res, err := sh.Context.clairClient.GetResult(layerName)
-	if err != nil {
-		logger.Errorf("Failed to get result from Clair, error: %v", err)
-		return "", err
+	if err := clair.UpdateScanOverview(sh.Context.Digest, layerName); err != nil {
+		return "", nil
 	}
-	vulnMap := make(map[models.Severity]int)
-	features := res.Layer.Features
-	totalComponents := len(features)
-	logger.Infof("total features: %d", totalComponents)
-	var temp models.Severity
-	for _, f := range features {
-		sev := models.SevNone
-		for _, v := range f.Vulnerabilities {
-			temp = clair.ParseClairSev(v.Severity)
-			if temp > sev {
-				sev = temp
-			}
-		}
-		logger.Infof("Feature: %s, Severity: %d", f.Name, sev)
-		vulnMap[sev]++
-	}
-	overallSev := models.SevNone
-	compSummary := []*models.ComponentsOverviewEntry{}
-	for k, v := range vulnMap {
-		if k > overallSev {
-			overallSev = k
-		}
-		entry := &models.ComponentsOverviewEntry{
-			Sev:   int(k),
-			Count: v,
-		}
-		compSummary = append(compSummary, entry)
-	}
-	compOverview := &models.ComponentsOverview{
-		Total:   totalComponents,
-		Summary: compSummary,
-	}
-	err = dao.UpdateImgScanOverview(sh.Context.Digest, layerName, overallSev, compOverview)
 	return models.JobFinished, nil
 }
 

src/ui/api/config.go

@@ -296,7 +296,7 @@ func validateCfg(c map[string]interface{}) (bool, error) {
 		scope != common.LDAPScopeBase &&
 		scope != common.LDAPScopeOnelevel &&
 		scope != common.LDAPScopeSubtree {
-		return false, fmt.Errorf("invalid %s, should be %s, %s or %s",
+		return false, fmt.Errorf("invalid %s, should be %d, %d or %d",
 			common.LDAPScope,
 			common.LDAPScopeBase,
 			common.LDAPScopeOnelevel,

src/ui/api/member.go

@@ -67,8 +67,7 @@ func (pma *ProjectMemberAPI) Prepare() {
 	}
 	project, err := pma.ProjectMgr.Get(pid)
 	if err != nil {
-		pma.HandleInternalServerError(
-			fmt.Sprintf("failed to get project %d: %v", pid, err))
+		pma.ParseAndHandleError(fmt.Sprintf("failed to get project %d", pid), err)
 		return
 	}
 	if project == nil {

src/ui/api/project.go

@@ -59,8 +59,7 @@ func (p *ProjectAPI) Prepare() {
 
 		project, err := p.ProjectMgr.Get(id)
 		if err != nil {
-			p.HandleInternalServerError(fmt.Sprintf("failed to get project %d: %v",
-				id, err))
+			p.ParseAndHandleError(fmt.Sprintf("failed to get project %d", id), err)
 			return
 		}
 
@@ -107,8 +106,8 @@ func (p *ProjectAPI) Post() {
 
 	exist, err := p.ProjectMgr.Exist(pro.Name)
 	if err != nil {
-		p.HandleInternalServerError(fmt.Sprintf("failed to check the existence of project %s: %v",
-			pro.Name, err))
+		p.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s",
+			pro.Name), err)
 		return
 	}
 	if exist {
@@ -126,12 +125,12 @@ func (p *ProjectAPI) Post() {
 		AutomaticallyScanImagesOnPush:              pro.AutomaticallyScanImagesOnPush,
 	})
 	if err != nil {
-		log.Errorf("Failed to add project, error: %v", err)
 		dup, _ := regexp.MatchString(dupProjectPattern, err.Error())
 		if dup {
+			log.Debugf("conflict %s", pro.Name)
 			p.RenderError(http.StatusConflict, "")
 		} else {
-			p.RenderError(http.StatusInternalServerError, "Failed to add project")
+			p.ParseAndHandleError("failed to add project", err)
 		}
 		return
 	}
@@ -163,8 +162,7 @@ func (p *ProjectAPI) Head() {
 
 	project, err := p.ProjectMgr.Get(name)
 	if err != nil {
-		p.HandleInternalServerError(fmt.Sprintf("failed to get project %s: %v",
-			name, err))
+		p.ParseAndHandleError(fmt.Sprintf("failed to get project %s", name), err)
 		return
 	}
 
@@ -223,8 +221,7 @@ func (p *ProjectAPI) Delete() {
 	}
 
 	if err = p.ProjectMgr.Delete(p.project.ProjectID); err != nil {
-		p.HandleInternalServerError(
-			fmt.Sprintf("failed to delete project %d: %v", p.project.ProjectID, err))
+		p.ParseAndHandleError(fmt.Sprintf("failed to delete project %d", p.project.ProjectID), err)
 		return
 	}
 
@@ -299,13 +296,13 @@ func (p *ProjectAPI) List() {
 
 	total, err := p.ProjectMgr.GetTotal(query, base)
 	if err != nil {
-		p.HandleInternalServerError(fmt.Sprintf("failed to get total of projects: %v", err))
+		p.ParseAndHandleError("failed to get total of projects", err)
 		return
 	}
 
 	projects, err := p.ProjectMgr.GetAll(query, base)
 	if err != nil {
-		p.HandleInternalServerError(fmt.Sprintf("failed to get projects: %v", err))
+		p.ParseAndHandleError("failed to get projects", err)
 		return
 	}
 
@@ -359,8 +356,8 @@ func (p *ProjectAPI) ToggleProjectPublic() {
 		&models.Project{
 			Public: req.Public,
 		}); err != nil {
-		p.HandleInternalServerError(fmt.Sprintf("failed to update project %d: %v",
-			p.project.ProjectID, err))
+		p.ParseAndHandleError(fmt.Sprintf("failed to update project %d",
+			p.project.ProjectID), err)
 		return
 	}
 }

src/ui/api/replication_policy.go

@@ -86,8 +86,8 @@ func (pa *RepPolicyAPI) List() {
 	for _, policy := range policies {
 		project, err := pa.ProjectMgr.Get(policy.ProjectID)
 		if err != nil {
-			pa.HandleInternalServerError(fmt.Sprintf(
-				"failed to get project %d: %v", policy.ProjectID, err))
+			pa.ParseAndHandleError(fmt.Sprintf(
+				"failed to get project %d", policy.ProjectID), err)
 			return
 		}
 		if project != nil {
@@ -118,8 +118,8 @@ func (pa *RepPolicyAPI) Post() {
 
 	project, err := pa.ProjectMgr.Get(policy.ProjectID)
 	if err != nil {
-		log.Errorf("failed to get project %d: %v", policy.ProjectID, err)
-		pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
+		pa.ParseAndHandleError(fmt.Sprintf("failed to get project %d", policy.ProjectID), err)
+		return
 	}
 
 	if project == nil {

src/ui/api/repository.go

@@ -84,8 +84,8 @@ func (ra *RepositoryAPI) Get() {
 
 	exist, err := ra.ProjectMgr.Exist(projectID)
 	if err != nil {
-		ra.HandleInternalServerError(fmt.Sprintf("failed to check the existence of project %d: %v",
-			projectID, err))
+		ra.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d",
+			projectID), err)
 		return
 	}
 
@@ -169,8 +169,8 @@ func (ra *RepositoryAPI) Delete() {
 	projectName, _ := utils.ParseRepository(repoName)
 	project, err := ra.ProjectMgr.Get(projectName)
 	if err != nil {
-		ra.HandleInternalServerError(fmt.Sprintf("failed to get the project %s: %v",
-			projectName, err))
+		ra.ParseAndHandleError(fmt.Sprintf("failed to get the project %s",
+			projectName), err)
 		return
 	}
 
@@ -200,7 +200,7 @@ func (ra *RepositoryAPI) Delete() {
 	if len(tag) == 0 {
 		tagList, err := rc.ListTag()
 		if err != nil {
-			if regErr, ok := err.(*registry_error.Error); ok {
+			if regErr, ok := err.(*registry_error.HTTPError); ok {
 				ra.CustomAbort(regErr.StatusCode, regErr.Detail)
 			}
 
@@ -242,7 +242,7 @@ func (ra *RepositoryAPI) Delete() {
 
 	for _, t := range tags {
 		if err = rc.DeleteTag(t); err != nil {
-			if regErr, ok := err.(*registry_error.Error); ok {
+			if regErr, ok := err.(*registry_error.HTTPError); ok {
 				if regErr.StatusCode == http.StatusNotFound {
 					continue
 				}
@@ -335,8 +335,8 @@ func (ra *RepositoryAPI) GetTags() {
 	projectName, _ := utils.ParseRepository(repoName)
 	exist, err := ra.ProjectMgr.Exist(projectName)
 	if err != nil {
-		ra.HandleInternalServerError(fmt.Sprintf("failed to check the existence of project %s: %v",
-			projectName, err))
+		ra.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s",
+			projectName), err)
 		return
 	}
 
@@ -475,8 +475,8 @@ func (ra *RepositoryAPI) GetManifests() {
 	projectName, _ := utils.ParseRepository(repoName)
 	exist, err := ra.ProjectMgr.Exist(projectName)
 	if err != nil {
-		ra.HandleInternalServerError(fmt.Sprintf("failed to check the existence of project %s: %v",
-			projectName, err))
+		ra.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s",
+			projectName), err)
 		return
 	}
 
@@ -503,7 +503,7 @@ func (ra *RepositoryAPI) GetManifests() {
 
 	manifest, err := getManifest(rc, tag, version)
 	if err != nil {
-		if regErr, ok := err.(*registry_error.Error); ok {
+		if regErr, ok := err.(*registry_error.HTTPError); ok {
 			ra.CustomAbort(regErr.StatusCode, regErr.Detail)
 		}
 
@@ -577,7 +577,7 @@ func (ra *RepositoryAPI) GetTopRepos() {
 	projectIDs := []int64{}
 	projects, err := ra.ProjectMgr.GetPublic()
 	if err != nil {
-		ra.HandleInternalServerError(fmt.Sprintf("failed to get public projects: %v", err))
+		ra.ParseAndHandleError("failed to get public projects", err)
 		return
 	}
 	if ra.SecurityCtx.IsAuthenticated() {
@@ -617,8 +617,8 @@ func (ra *RepositoryAPI) GetSignatures() {
 	projectName, _ := utils.ParseRepository(repoName)
 	exist, err := ra.ProjectMgr.Exist(projectName)
 	if err != nil {
-		ra.HandleInternalServerError(fmt.Sprintf("failed to check the existence of project %s: %v",
-			projectName, err))
+		ra.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s",
+			projectName), err)
 		return
 	}
 
@@ -658,8 +658,8 @@ func (ra *RepositoryAPI) ScanImage() {
 	projectName, _ := utils.ParseRepository(repoName)
 	exist, err := ra.ProjectMgr.Exist(projectName)
 	if err != nil {
-		ra.HandleInternalServerError(fmt.Sprintf("failed to check the existence of project %s: %v",
-			projectName, err))
+		ra.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s",
+			projectName), err)
 		return
 	}
 	if !exist {
@@ -745,6 +745,13 @@ func (ra *RepositoryAPI) ScanAll() {
 		ra.HandleForbidden(ra.SecurityCtx.GetUsername())
 		return
 	}
+
+	if !utils.ScanAllMarker().Mark() {
+		log.Warningf("There is a scan all scheduled at: %v, the request will not be processed.", utils.ScanAllMarker().Next())
+		ra.RenderError(http.StatusPreconditionFailed, "Unable handle frequent scan all requests")
+		return
+	}
+
 	if err := uiutils.ScanAllImages(); err != nil {
 		log.Errorf("Failed triggering scan all images, error: %v", err)
 		ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err))
@@ -776,7 +783,7 @@ func (ra *RepositoryAPI) checkExistence(repository, tag string) (bool, string, e
 	project, _ := utils.ParseRepository(repository)
 	exist, err := ra.ProjectMgr.Exist(project)
 	if err != nil {
-		return false, "", fmt.Errorf("failed to check the existence of project %s: %v", project, err)
+		return false, "", err
 	}
 	if !exist {
 		log.Errorf("project %s not found", project)

src/ui/api/search.go

@@ -51,15 +51,13 @@ func (s *SearchAPI) Get() {
 	if isSysAdmin {
 		projects, err = s.ProjectMgr.GetAll(nil)
 		if err != nil {
-			s.HandleInternalServerError(fmt.Sprintf(
-				"failed to get projects: %v", err))
+			s.ParseAndHandleError("failed to get projects", err)
 			return
 		}
 	} else {
 		projects, err = s.ProjectMgr.GetPublic()
 		if err != nil {
-			s.HandleInternalServerError(fmt.Sprintf(
-				"failed to get projects: %v", err))
+			s.ParseAndHandleError("failed to get projects", err)
 			return
 		}
 		if isAuthenticated {

src/ui/api/statistic.go

@@ -59,8 +59,7 @@ func (s *StatisticAPI) Get() {
 	statistic := map[string]int64{}
 	pubProjs, err := s.ProjectMgr.GetPublic()
 	if err != nil {
-		s.HandleInternalServerError(fmt.Sprintf(
-			"failed to get public projects: %v", err))
+		s.ParseAndHandleError("failed to get public projects", err)
 		return
 	}
 
@@ -102,8 +101,8 @@ func (s *StatisticAPI) Get() {
 			},
 		})
 		if err != nil {
-			s.HandleInternalServerError(fmt.Sprintf(
-				"failed to get projects of user %s: %v", s.username, err))
+			s.ParseAndHandleError(fmt.Sprintf(
+				"failed to get projects of user %s", s.username), err)
 			return
 		}
 

src/ui/api/systeminfo.go

@@ -19,8 +19,11 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/vmware/harbor/src/common"
+	"github.com/vmware/harbor/src/common/dao"
+	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/ui/config"
 )
@@ -46,16 +49,17 @@ type Storage struct {
 
 //GeneralInfo wraps common systeminfo for anonymous request
 type GeneralInfo struct {
-	WithNotary              bool   `json:"with_notary"`
-	WithClair               bool   `json:"with_clair"`
-	WithAdmiral             bool   `json:"with_admiral"`
-	AdmiralEndpoint         string `json:"admiral_endpoint"`
-	AuthMode                string `json:"auth_mode"`
-	RegistryURL             string `json:"registry_url"`
-	ProjectCreationRestrict string `json:"project_creation_restriction"`
-	SelfRegistration        bool   `json:"self_registration"`
-	HasCARoot               bool   `json:"has_ca_root"`
-	HarborVersion           string `json:"harbor_version"`
+	WithNotary              bool                             `json:"with_notary"`
+	WithClair               bool                             `json:"with_clair"`
+	WithAdmiral             bool                             `json:"with_admiral"`
+	AdmiralEndpoint         string                           `json:"admiral_endpoint"`
+	AuthMode                string                           `json:"auth_mode"`
+	RegistryURL             string                           `json:"registry_url"`
+	ProjectCreationRestrict string                           `json:"project_creation_restriction"`
+	SelfRegistration        bool                             `json:"self_registration"`
+	HasCARoot               bool                             `json:"has_ca_root"`
+	HarborVersion           string                           `json:"harbor_version"`
+	ClairVulnStatus         *models.ClairVulnerabilityStatus `json:"clair_vulnerability_status,omitempty"`
 }
 
 // validate for validating user if an admin.
@@ -134,11 +138,14 @@ func (sia *SystemInfoAPI) GetGeneralInfo() {
 		HasCARoot:               caStatErr == nil,
 		HarborVersion:           harborVersion,
 	}
+	if info.WithClair {
+		info.ClairVulnStatus = getClairVulnStatus()
+	}
 	sia.Data["json"] = info
 	sia.ServeJSON()
 }
 
-// GetVersion gets harbor version.
+// getVersion gets harbor version.
 func (sia *SystemInfoAPI) getVersion() string {
 	version, err := ioutil.ReadFile(harborVersionFile)
 	if err != nil {
@@ -147,3 +154,34 @@ func (sia *SystemInfoAPI) getVersion() string {
 	}
 	return string(version[:])
 }
+
+func getClairVulnStatus() *models.ClairVulnerabilityStatus {
+	res := &models.ClairVulnerabilityStatus{}
+	l, err := dao.ListClairVulnTimestamps()
+	if err != nil {
+		log.Errorf("Failed to list Clair vulnerability timestamps, error:%v", err)
+		return nil
+	}
+	m := make(map[string]time.Time)
+	var t time.Time
+	for _, e := range l {
+		if e.LastUpdate.After(t) {
+			t = e.LastUpdate
+		}
+		ns := strings.Split(e.Namespace, ":")
+		if ts, ok := m[ns[0]]; !ok || ts.Before(e.LastUpdate) {
+			m[ns[0]] = e.LastUpdate
+		}
+	}
+	res.Overall = &t
+	details := []models.ClairNamespaceTimestamp{}
+	for k, v := range m {
+		e := models.ClairNamespaceTimestamp{
+			Namespace: k,
+			Timestamp: v,
+		}
+		details = append(details, e)
+	}
+	res.Details = details
+	return res
+}

src/ui/api/target.go

@@ -81,7 +81,7 @@ func (t *TargetAPI) ping(endpoint, username, password string) {
 	}
 
 	if err = registry.Ping(); err != nil {
-		if regErr, ok := err.(*registry_error.Error); ok {
+		if regErr, ok := err.(*registry_error.HTTPError); ok {
 			t.CustomAbort(regErr.StatusCode, regErr.Detail)
 		}
 

src/ui/api/utils.go

@@ -443,7 +443,7 @@ func getReposByProject(name string, keyword ...string) ([]string, error) {
 func repositoryExist(name string, client *registry.Repository) (bool, error) {
 	tags, err := client.ListTag()
 	if err != nil {
-		if regErr, ok := err.(*registry_error.Error); ok && regErr.StatusCode == http.StatusNotFound {
+		if regErr, ok := err.(*registry_error.HTTPError); ok && regErr.StatusCode == http.StatusNotFound {
 			return false, nil
 		}
 		return false, err

src/ui/config/config.go

@@ -355,7 +355,7 @@ func WithClair() bool {
 
 // ClairEndpoint returns the end point of clair instance, by default it's the one deployed within Harbor.
 func ClairEndpoint() string {
-	return "http://clair:6060"
+	return common.DefaultClairEndpoint
 }
 
 // AdmiralEndpoint returns the URL of admiral, if Harbor is not deployed with admiral it should return an empty string.

src/ui/projectmanager/pms/pm.go

@@ -412,7 +412,7 @@ func (p *ProjectManager) send(method, path string, body io.Reader) ([]byte, erro
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, &er.Error{
+		return nil, &er.HTTPError{
 			StatusCode: resp.StatusCode,
 			Detail:     string(b),
 		}

src/ui/service/notifications/clair/handler.go

@@ -16,11 +16,11 @@ package clair
 
 import (
 	"encoding/json"
-	"sync"
 	"time"
 
 	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils"
 	"github.com/vmware/harbor/src/common/utils/clair"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/ui/api"
@@ -31,24 +31,7 @@ const (
 	rescanInterval = 15 * time.Minute
 )
 
-type timer struct {
-	sync.Mutex
-	next time.Time
-}
-
-// returns true to indicate it should reshedule the "rescan" action.
-func (t *timer) needReschedule() bool {
-	t.Lock()
-	defer t.Unlock()
-	if time.Now().Before(t.next) {
-		return false
-	}
-	t.next = time.Now().Add(rescanInterval)
-	return true
-}
-
 var (
-	rescanTimer = timer{}
 	clairClient = clair.NewClient(config.ClairEndpoint(), nil)
 )
 
@@ -93,13 +76,24 @@ func (h *Handler) Handle() {
 			}
 		}
 	}
-	if rescanTimer.needReschedule() {
+	if utils.ScanOverviewMarker().Mark() {
 		go func() {
 			<-time.After(rescanInterval)
-			log.Debugf("TODO: rescan or resfresh scan_overview!")
+			l, err := dao.ListImgScanOverviews()
+			if err != nil {
+				log.Errorf("Failed to list scan overview records, error: %v", err)
+				return
+			}
+			for _, e := range l {
+				if err := clair.UpdateScanOverview(e.Digest, e.DetailsKey); err != nil {
+					log.Errorf("Failed to refresh scan overview for image: %s", e.Digest)
+				} else {
+					log.Debugf("Refreshed scan overview for record with digest: %s", e.Digest)
+				}
+			}
 		}()
 	} else {
-		log.Debugf("There is a rescan scheduled already, skip.")
+		log.Debugf("There is a rescan scheduled at %v already, skip.", utils.ScanOverviewMarker().Next())
 	}
 	if err := clairClient.DeleteNotification(ne.Notification.Name); err != nil {
 		log.Warningf("Failed to remove notification from Clair, name: %s", ne.Notification.Name)

src/ui_ng/lib/src/tag/tag.component.ts

@@ -167,7 +167,7 @@ export class TagComponent implements OnInit, OnDestroy {
         this.loading = false;
       });
     let hnd = setInterval(() => this.ref.markForCheck(), 100);
-    setTimeout(() => clearInterval(hnd), 1000);
+    setTimeout(() => clearInterval(hnd), 5000);
   }
 
   deleteTag(tag: Tag) {
@@ -272,4 +272,4 @@ export class TagComponent implements OnInit, OnDestroy {
       this.textInput.nativeElement.select();
     }
   }
-}
+}

src/vendor/github.com/astaxie/beego/orm/orm.go

@@ -137,11 +137,10 @@ func (o *orm) ReadOrCreate(md interface{}, col1 string, cols ...string) (bool, i
 	if err == ErrNoRows {
 		// Create
 		id, err := o.Insert(md)
-		fmt.Printf("id when create: %d", id)
 		return (err == nil), id, err
 	}
 
-	return false, 0, err
+	return false, ind.FieldByIndex(mi.fields.pk.fieldIndex).Int(), err
 }
 
 // insert model data to database