Merge pull request #2730 from reasonerjt/scan-all-control

restrict scan all API, it should be called once every a period of time

src/common/utils/timemarker.go

@@ -0,0 +1,78 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"os"
+	"strconv"
+	"sync"
+	"time"
+)
+
+var (
+	scanAllMarker      *TimeMarker
+	scanOverviewMarker = &TimeMarker{
+		interval: 15 * time.Second,
+	}
+	once sync.Once
+)
+
+//TimeMarker is used to control an action not to be taken frequently within the interval
+type TimeMarker struct {
+	sync.RWMutex
+	next     time.Time
+	interval time.Duration
+}
+
+//Mark tries to mark a future time, which is after the duration of interval from the time it's called.
+//It returns false if there is a mark in fugure, true if the mark is successfully set.
+func (t *TimeMarker) Mark() bool {
+	t.Lock()
+	defer t.Unlock()
+	if time.Now().Before(t.next) {
+		return false
+	}
+	t.next = time.Now().Add(t.interval)
+	return true
+}
+
+//Next returns the time of the next mark.
+func (t *TimeMarker) Next() time.Time {
+	t.RLock()
+	defer t.RUnlock()
+	return t.next
+}
+
+//ScanAllMarker ...
+func ScanAllMarker() *TimeMarker {
+	once.Do(func() {
+		a := os.Getenv("HARBOR_SCAN_ALL_INTERVAL")
+		if m, err := strconv.Atoi(a); err == nil {
+			scanAllMarker = &TimeMarker{
+				interval: time.Duration(m) * time.Minute,
+			}
+		} else {
+			scanAllMarker = &TimeMarker{
+				interval: 30 * time.Minute,
+			}
+		}
+	})
+	return scanAllMarker
+}
+
+//ScanOverviewMarker ...
+func ScanOverviewMarker() *TimeMarker {
+	return scanOverviewMarker
+}

src/common/utils/timemarker_test.go

@@ -0,0 +1,48 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestTimeMarker(t *testing.T) {
+	assert := assert.New(t)
+	m := &TimeMarker{
+		interval: 1 * time.Second,
+	}
+	r1 := m.Mark()
+	assert.True(r1)
+	r2 := m.Mark()
+	assert.False(r2)
+	t.Log("Sleep for 2 seconds...")
+	time.Sleep(2 * time.Second)
+	r3 := m.Mark()
+	assert.True(r3)
+}
+
+func TestScanMarkers(t *testing.T) {
+	assert := assert.New(t)
+	os.Setenv("HARBOR_SCAN_ALL_INTERVAL", "5")
+	sm := ScanAllMarker()
+	d := sm.Next().Sub(time.Now())
+	assert.True(d <= 5*time.Minute)
+	som := ScanOverviewMarker()
+	d = som.Next().Sub(time.Now())
+	assert.True(d <= 15*time.Second)
+}

src/ui/api/repository.go

@@ -745,6 +745,13 @@ func (ra *RepositoryAPI) ScanAll() {
 		ra.HandleForbidden(ra.SecurityCtx.GetUsername())
 		return
 	}
+
+	if !utils.ScanAllMarker().Mark() {
+		log.Warningf("There is a scan all scheduled at: %v, the request will not be processed.", utils.ScanAllMarker().Next())
+		ra.RenderError(http.StatusPreconditionFailed, "Unable handle frequent scan all requests")
+		return
+	}
+
 	if err := uiutils.ScanAllImages(); err != nil {
 		log.Errorf("Failed triggering scan all images, error: %v", err)
 		ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err))

src/ui/service/notifications/clair/handler.go

@@ -16,11 +16,11 @@ package clair
 
 import (
 	"encoding/json"
-	"sync"
 	"time"
 
 	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils"
 	"github.com/vmware/harbor/src/common/utils/clair"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/ui/api"
@@ -31,24 +31,7 @@ const (
 	rescanInterval = 15 * time.Minute
 )
 
-type timer struct {
-	sync.Mutex
-	next time.Time
-}
-
-// returns true to indicate it should reshedule the "rescan" action.
-func (t *timer) needReschedule() bool {
-	t.Lock()
-	defer t.Unlock()
-	if time.Now().Before(t.next) {
-		return false
-	}
-	t.next = time.Now().Add(rescanInterval)
-	return true
-}
-
 var (
-	rescanTimer = timer{}
 	clairClient = clair.NewClient(config.ClairEndpoint(), nil)
 )
 
@@ -93,7 +76,7 @@ func (h *Handler) Handle() {
 			}
 		}
 	}
-	if rescanTimer.needReschedule() {
+	if utils.ScanOverviewMarker().Mark() {
 		go func() {
 			<-time.After(rescanInterval)
 			l, err := dao.ListImgScanOverviews()
@@ -110,7 +93,7 @@ func (h *Handler) Handle() {
 			}
 		}()
 	} else {
-		log.Debugf("There is a rescan scheduled already, skip.")
+		log.Debugf("There is a rescan scheduled at %v already, skip.", utils.ScanOverviewMarker().Next())
 	}
 	if err := clairClient.DeleteNotification(ne.Notification.Name); err != nil {
 		log.Warningf("Failed to remove notification from Clair, name: %s", ne.Notification.Name)