Merge remote-tracking branch 'upstream/master' into 170628_getpm

Conflicts:
	src/ui/config/config.go
	src/ui/proxy/interceptor_test.go

make/common/templates/clair/postgresql-init.d/README.md

@@ -0,0 +1,7 @@
+This folder used to run some initial sql for clair if needed.
+
+Just put the sql file in this directory and then start the
+clair . 
+
+both .sql and .gz format supported
+

make/docker-compose.clair.yml

@@ -24,6 +24,7 @@ services:
     env_file:
       ./common/config/clair/postgres_env
     volumes:
+      - ./common/config/clair/postgresql-init.d/:/docker-entrypoint-initdb.d
       - /data/clair-db:/var/lib/postgresql/data
     logging:
       driver: "syslog"

make/prepare

@@ -373,6 +373,10 @@ if args.clair_mode:
     pg_password = "password"
     clair_temp_dir = os.path.join(templates_dir, "clair")
     clair_config_dir = prep_conf_dir(config_dir, "clair")
+    print("Copying offline data file for clair DB")
+    if os.path.exists(os.path.join(clair_config_dir, "postgresql-init.d")):
+        shutil.rmtree(os.path.join(clair_config_dir, "postgresql-init.d"))
+    shutil.copytree(os.path.join(clair_temp_dir, "postgresql-init.d"), os.path.join(clair_config_dir, "postgresql-init.d"))
     postgres_env = os.path.join(clair_config_dir, "postgres_env") 
     render(os.path.join(clair_temp_dir, "postgres_env"), postgres_env, password = pg_password)
     clair_conf = os.path.join(clair_config_dir, "config.yaml")

src/common/const.go

@@ -20,9 +20,9 @@ const (
 	LDAPAuth            = "ldap_auth"
 	ProCrtRestrEveryone = "everyone"
 	ProCrtRestrAdmOnly  = "adminonly"
-	LDAPScopeBase       = "1"
+	LDAPScopeBase       = 1
-	LDAPScopeOnelevel   = "2"
+	LDAPScopeOnelevel   = 2
-	LDAPScopeSubtree    = "3"
+	LDAPScopeSubtree    = 3
 
 	RoleProjectAdmin = 1
 	RoleDeveloper    = 2
@@ -65,4 +65,5 @@ const (
 	AdmiralEndpoint            = "admiral_url"
 	WithNotary                 = "with_notary"
 	WithClair                  = "with_clair"
+	ScanAllPolicy              = "scan_all_policy"
 )

src/common/models/scan_job.go

@@ -96,3 +96,28 @@ type VulnerabilityItem struct {
 	Description string   `json:"description"`
 	Fixed       string   `json:"fixedVersion,omitempty"`
 }
+
+// ScanAllPolicy is represent the json request and object for scan all policy, the parm is het
+type ScanAllPolicy struct {
+	Type string                 `json:"type"`
+	Parm map[string]interface{} `json:"parameter, omitempty"`
+}
+
+const (
+	// ScanAllNone "none" for not doing any scan all
+	ScanAllNone = "none"
+	// ScanAllDaily for doing scan all daily
+	ScanAllDaily = "daily"
+	// ScanAllOnRefresh for doing scan all when the Clair DB is refreshed.
+	ScanAllOnRefresh = "on_refresh"
+	// ScanAllDailyTime the key for parm of daily scan all policy.
+	ScanAllDailyTime = "daily_time"
+)
+
+//DefaultScanAllPolicy ...
+var DefaultScanAllPolicy = ScanAllPolicy{
+	Type: ScanAllDaily,
+	Parm: map[string]interface{}{
+		ScanAllDailyTime: 0,
+	},
+}

src/common/utils/registry/auth/tokenauthorizer.go

@@ -15,20 +15,13 @@
 package auth
 
 import (
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
-	"net/url"
 	"strings"
 	"sync"
 	"time"
 
-	//"github.com/vmware/harbor/src/common/config"
-	"github.com/vmware/harbor/src/common/models"
-	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/common/utils/registry"
-	registry_error "github.com/vmware/harbor/src/common/utils/error"
 	token_util "github.com/vmware/harbor/src/ui/service/token"
 )
 
@@ -36,13 +29,14 @@ const (
 	latency int = 10 //second, the network latency when token is received
 )
 
-type scope struct {
+// Scope ...
+type Scope struct {
 	Type    string
 	Name    string
 	Actions []string
 }
 
-func (s *scope) string() string {
+func (s *Scope) string() string {
 	return fmt.Sprintf("%s:%s:%s", s.Type, s.Name, strings.Join(s.Actions, ","))
 }
 
@@ -50,7 +44,7 @@ type tokenGenerator func(realm, service string, scopes []string) (token string,
 
 // Implements interface Authorizer
 type tokenAuthorizer struct {
-	scope     *scope
+	scope     *Scope
 	tg        tokenGenerator
 	cache     string     // cached token
 	expiresAt *time.Time // The UTC standard time at when the token will expire
@@ -64,13 +58,13 @@ func (t *tokenAuthorizer) Scheme() string {
 
 // AuthorizeRequest will add authorization header which contains a token before the request is sent
 func (t *tokenAuthorizer) Authorize(req *http.Request, params map[string]string) error {
-	var scopes []*scope
+	var scopes []*Scope
 	var token string
 
 	hasFrom := false
 	from := req.URL.Query().Get("from")
 	if len(from) != 0 {
-		s := &scope{
+		s := &Scope{
 			Type:    "repository",
 			Name:    from,
 			Actions: []string{"pull"},
@@ -154,7 +148,7 @@ func NewStandardTokenAuthorizer(credential Credential, insecure bool,
 	}
 
 	if len(scopeType) != 0 || len(scopeName) != 0 {
-		authorizer.scope = &scope{
+		authorizer.scope = &Scope{
 			Type:    scopeType,
 			Name:    scopeName,
 			Actions: scopeActions,
@@ -166,66 +160,21 @@ func NewStandardTokenAuthorizer(credential Credential, insecure bool,
 	return authorizer
 }
 
-func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
+func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes []string) (string, int, *time.Time, error) {
 	realm = s.tokenURL(realm)
+	tk, err := getToken(s.client, s.credential, realm,
+		service, scopes)
 
-	u, err := url.Parse(realm)
+	if len(tk.IssuedAt) == 0 {
+		return tk.Token, tk.ExpiresIn, nil, nil
+	}
+
+	issuedAt, err := time.Parse(time.RFC3339, tk.IssuedAt)
 	if err != nil {
-		return
+		return "", 0, nil, err
-	}
-	q := u.Query()
-	q.Add("service", service)
-	for _, scope := range scopes {
-		q.Add("scope", scope)
-	}
-	u.RawQuery = q.Encode()
-	r, err := http.NewRequest("GET", u.String(), nil)
-	if err != nil {
-		return
 	}
 
-	if s.credential != nil {
+	return tk.Token, tk.ExpiresIn, &issuedAt, nil
-		s.credential.AddAuthorization(r)
-	}
-
-	resp, err := s.client.Do(r)
-	if err != nil {
-		return
-	}
-
-	defer resp.Body.Close()
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return
-	}
-	if resp.StatusCode != http.StatusOK {
-		err = &registry_error.Error{
-			StatusCode: resp.StatusCode,
-			Detail:     string(b),
-		}
-		return
-	}
-
-	tk := models.Token{}
-	if err = json.Unmarshal(b, &tk); err != nil {
-		return
-	}
-
-	token = tk.Token
-
-	expiresIn = tk.ExpiresIn
-
-	if len(tk.IssuedAt) != 0 {
-		t, err := time.Parse(time.RFC3339, tk.IssuedAt)
-		if err != nil {
-			log.Errorf("error occurred while parsing issued_at: %v", err)
-			err = nil
-		} else {
-			issuedAt = &t
-		}
-	}
-
-	return
 }
 
 // when the registry client is used inside Harbor, the token request
@@ -267,7 +216,7 @@ func newUsernameTokenAuthorizer(notary bool, username, scopeType, scopeName stri
 		username: username,
 	}
 
-	authorizer.scope = &scope{
+	authorizer.scope = &Scope{
 		Type:    scopeType,
 		Name:    scopeName,
 		Actions: scopeActions,

src/common/utils/registry/auth/util.go

@@ -0,0 +1,93 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+
+	"github.com/vmware/harbor/src/common/models"
+
+	registry_error "github.com/vmware/harbor/src/common/utils/error"
+	"github.com/vmware/harbor/src/common/utils/registry"
+)
+
+const (
+	service = "harbor-registry"
+)
+
+// GetToken requests a token against the endpoint using credetial provided
+func GetToken(endpoint string, insecure bool, credential Credential,
+	scopes []*Scope) (*models.Token, error) {
+	client := &http.Client{
+		Transport: registry.GetHTTPTransport(insecure),
+	}
+
+	scopesStr := []string{}
+	for _, scope := range scopes {
+		scopesStr = append(scopesStr, scope.string())
+	}
+
+	return getToken(client, credential, endpoint, service, scopesStr)
+}
+
+func getToken(client *http.Client, credential Credential, realm, service string,
+	scopes []string) (*models.Token, error) {
+	u, err := url.Parse(realm)
+	if err != nil {
+		return nil, err
+	}
+	query := u.Query()
+	query.Add("service", service)
+	for _, scope := range scopes {
+		query.Add("scope", scope)
+	}
+	u.RawQuery = query.Encode()
+
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if credential != nil {
+		credential.AddAuthorization(req)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, &registry_error.Error{
+			StatusCode: resp.StatusCode,
+			Detail:     string(data),
+		}
+	}
+
+	token := &models.Token{}
+	if err = json.Unmarshal(data, token); err != nil {
+		return nil, err
+	}
+
+	return token, nil
+}

src/jobservice/utils/utils.go

@@ -15,11 +15,8 @@
 package utils
 
 import (
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
-	"net/url"
 
 	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/utils/registry"
@@ -64,39 +61,18 @@ func BuildBlobURL(endpoint, repository, digest string) string {
 	return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repository, digest)
 }
 
-//GetTokenForRepo is a temp solution for job handler to get a token for clair.
+//GetTokenForRepo is used for job handler to get a token for clair.
-//TODO: Get rid of it when it can get a token from repository client.
 func GetTokenForRepo(repository string) (string, error) {
-	u, err := url.Parse(config.InternalTokenServiceEndpoint())
-	if err != nil {
-		return "", err
-	}
-	q := u.Query()
-	q.Add("service", "harbor-registry")
-	q.Add("scope", fmt.Sprintf("repository:%s:pull", repository))
-	u.RawQuery = q.Encode()
-	r, err := http.NewRequest("GET", u.String(), nil)
-	if err != nil {
-		return "", err
-	}
 	c := &http.Cookie{Name: models.UISecretCookie, Value: config.JobserviceSecret()}
-	r.AddCookie(c)
+	credentail := auth.NewCookieCredential(c)
-	client := &http.Client{}
+	token, err := auth.GetToken(config.InternalTokenServiceEndpoint(), true, credentail, []*auth.Scope{&auth.Scope{
-	resp, err := client.Do(r)
+		Type:    "repository",
+		Name:    repository,
+		Actions: []string{"pull"},
+	}})
 	if err != nil {
 		return "", err
 	}
-	defer resp.Body.Close()
+
-	b, err := ioutil.ReadAll(resp.Body)
+	return token.Token, nil
-	if err != nil {
-		return "", err
-	}
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("Unexpected response from token service, code: %d, %s", resp.StatusCode, string(b))
-	}
-	tk := models.Token{}
-	if err := json.Unmarshal(b, &tk); err != nil {
-		return "", err
-	}
-	return tk.Token, nil
 }

src/ui/api/config.go

@@ -17,10 +17,11 @@ package api
 import (
 	"fmt"
 	"net/http"
-	"strconv"
+	"reflect"
 
 	"github.com/vmware/harbor/src/common"
 	"github.com/vmware/harbor/src/common/dao"
+	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/ui/config"
 )
@@ -62,6 +63,34 @@ var (
 		common.CfgExpiration,
 		common.JobLogDir,
 		common.AdminInitialPassword,
+		common.ScanAllPolicy,
+	}
+
+	stringKeys = []string{
+		common.ExtEndpoint,
+		common.AUTHMode,
+		common.DatabaseType,
+		common.MySQLHost,
+		common.MySQLUsername,
+		common.MySQLPassword,
+		common.MySQLDatabase,
+		common.SQLiteFile,
+		common.LDAPURL,
+		common.LDAPSearchDN,
+		common.LDAPSearchPwd,
+		common.LDAPBaseDN,
+		common.LDAPUID,
+		common.LDAPFilter,
+		common.TokenServiceURL,
+		common.RegistryURL,
+		common.EmailHost,
+		common.EmailUsername,
+		common.EmailPassword,
+		common.EmailFrom,
+		common.EmailIdentity,
+		common.ProjectCreationRestriction,
+		common.JobLogDir,
+		common.AdminInitialPassword,
 	}
 
 	numKeys = []string{
@@ -131,10 +160,10 @@ func (c *ConfigAPI) Get() {
 
 // Put updates configurations
 func (c *ConfigAPI) Put() {
-	m := map[string]string{}
+	m := map[string]interface{}{}
 	c.DecodeJSONReq(&m)
 
-	cfg := map[string]string{}
+	cfg := map[string]interface{}{}
 	for _, k := range validKeys {
 		if v, ok := m[k]; ok {
 			cfg[k] = v
@@ -152,35 +181,7 @@ func (c *ConfigAPI) Put() {
 		c.CustomAbort(http.StatusBadRequest, err.Error())
 	}
 
-	if value, ok := cfg[common.AUTHMode]; ok {
+	if err := config.Upload(cfg); err != nil {
-		mode, err := config.AuthMode()
-		if err != nil {
-			log.Errorf("failed to get auth mode: %v", err)
-			c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
-		}
-
-		if mode != value {
-			flag, err := authModeCanBeModified()
-			if err != nil {
-				log.Errorf("failed to determine whether auth mode can be modified: %v", err)
-				c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
-			}
-
-			if !flag {
-				c.CustomAbort(http.StatusBadRequest,
-					fmt.Sprintf("%s can not be modified as new users have been inserted into database",
-						common.AUTHMode))
-			}
-		}
-	}
-
-	result, err := convertForPut(cfg)
-	if err != nil {
-		log.Errorf("failed to convert configurations: %v", err)
-		c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
-	}
-
-	if err := config.Upload(result); err != nil {
 		log.Errorf("failed to upload configurations: %v", err)
 		c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
 	}
@@ -199,18 +200,53 @@ func (c *ConfigAPI) Reset() {
 	}
 }
 
-func validateCfg(c map[string]string) (bool, error) {
+func validateCfg(c map[string]interface{}) (bool, error) {
-	isSysErr := false
+	strMap := map[string]string{}
+	for _, k := range stringKeys {
+		if _, ok := c[k]; !ok {
+			continue
+		}
+		if _, ok := c[k].(string); !ok {
+			return false, fmt.Errorf("Invalid value type, expected string, key: %s, value: %v, type: %v", k, c[k], reflect.TypeOf(c[k]))
+		}
+		strMap[k] = c[k].(string)
+	}
+	numMap := map[string]int{}
+	for _, k := range numKeys {
+		if _, ok := c[k]; !ok {
+			continue
+		}
+		if _, ok := c[k].(float64); !ok {
+			return false, fmt.Errorf("Invalid value type, expected float64, key: %s, value: %v, type: %v", k, c[k], reflect.TypeOf(c[k]))
+		}
+		numMap[k] = int(c[k].(float64))
+	}
+	boolMap := map[string]bool{}
+	for _, k := range boolKeys {
+		if _, ok := c[k]; !ok {
+			continue
+		}
+		if _, ok := c[k].(bool); !ok {
+			return false, fmt.Errorf("Invalid value type, expected bool, key: %s, value: %v, type: %v", k, c[k], reflect.TypeOf(c[k]))
+		}
+		boolMap[k] = c[k].(bool)
+	}
 
 	mode, err := config.AuthMode()
 	if err != nil {
-		isSysErr = true
+		return true, err
-		return isSysErr, err
 	}
 
-	if value, ok := c[common.AUTHMode]; ok {
+	if value, ok := strMap[common.AUTHMode]; ok {
 		if value != common.DBAuth && value != common.LDAPAuth {
-			return isSysErr, fmt.Errorf("invalid %s, shoud be %s or %s", common.AUTHMode, common.DBAuth, common.LDAPAuth)
+			return false, fmt.Errorf("invalid %s, shoud be %s or %s", common.AUTHMode, common.DBAuth, common.LDAPAuth)
+		}
+		flag, err := authModeCanBeModified()
+		if err != nil {
+			return true, err
+		}
+		if mode != value && !flag {
+			return false, fmt.Errorf("%s can not be modified as new users have been inserted into database", common.AUTHMode)
 		}
 		mode = value
 	}
@@ -218,123 +254,70 @@ func validateCfg(c map[string]string) (bool, error) {
 	if mode == common.LDAPAuth {
 		ldap, err := config.LDAP()
 		if err != nil {
-			isSysErr = true
+			return true, err
-			return isSysErr, err
 		}
 
 		if len(ldap.URL) == 0 {
-			if _, ok := c[common.LDAPURL]; !ok {
+			if _, ok := strMap[common.LDAPURL]; !ok {
-				return isSysErr, fmt.Errorf("%s is missing", common.LDAPURL)
+				return false, fmt.Errorf("%s is missing", common.LDAPURL)
 			}
 		}
 
 		if len(ldap.BaseDN) == 0 {
-			if _, ok := c[common.LDAPBaseDN]; !ok {
+			if _, ok := strMap[common.LDAPBaseDN]; !ok {
-				return isSysErr, fmt.Errorf("%s is missing", common.LDAPBaseDN)
+				return false, fmt.Errorf("%s is missing", common.LDAPBaseDN)
 			}
 		}
 		if len(ldap.UID) == 0 {
-			if _, ok := c[common.LDAPUID]; !ok {
+			if _, ok := strMap[common.LDAPUID]; !ok {
-				return isSysErr, fmt.Errorf("%s is missing", common.LDAPUID)
+				return false, fmt.Errorf("%s is missing", common.LDAPUID)
 			}
 		}
 		if ldap.Scope == 0 {
-			if _, ok := c[common.LDAPScope]; !ok {
+			if _, ok := numMap[common.LDAPScope]; !ok {
-				return isSysErr, fmt.Errorf("%s is missing", common.LDAPScope)
+				return false, fmt.Errorf("%s is missing", common.LDAPScope)
 			}
 		}
 	}
 
-	if ldapURL, ok := c[common.LDAPURL]; ok && len(ldapURL) == 0 {
+	if ldapURL, ok := strMap[common.LDAPURL]; ok && len(ldapURL) == 0 {
-		return isSysErr, fmt.Errorf("%s is empty", common.LDAPURL)
+		return false, fmt.Errorf("%s is empty", common.LDAPURL)
 	}
-	if baseDN, ok := c[common.LDAPBaseDN]; ok && len(baseDN) == 0 {
+	if baseDN, ok := strMap[common.LDAPBaseDN]; ok && len(baseDN) == 0 {
-		return isSysErr, fmt.Errorf("%s is empty", common.LDAPBaseDN)
+		return false, fmt.Errorf("%s is empty", common.LDAPBaseDN)
 	}
-	if uID, ok := c[common.LDAPUID]; ok && len(uID) == 0 {
+	if uID, ok := strMap[common.LDAPUID]; ok && len(uID) == 0 {
-		return isSysErr, fmt.Errorf("%s is empty", common.LDAPUID)
+		return false, fmt.Errorf("%s is empty", common.LDAPUID)
 	}
-	if scope, ok := c[common.LDAPScope]; ok &&
+	if scope, ok := numMap[common.LDAPScope]; ok &&
 		scope != common.LDAPScopeBase &&
 		scope != common.LDAPScopeOnelevel &&
 		scope != common.LDAPScopeSubtree {
-		return isSysErr, fmt.Errorf("invalid %s, should be %s, %s or %s",
+		return false, fmt.Errorf("invalid %s, should be %s, %s or %s",
 			common.LDAPScope,
 			common.LDAPScopeBase,
 			common.LDAPScopeOnelevel,
 			common.LDAPScopeSubtree)
 	}
-
+	for k, n := range numMap {
-	for _, k := range boolKeys {
+		if n < 0 {
-		v, ok := c[k]
+			return false, fmt.Errorf("invalid %s: %d", k, n)
-		if !ok {
-			continue
 		}
-
-		if v != "0" && v != "1" {
-			return isSysErr, fmt.Errorf("%s should be %s or %s",
-				k, "0", "1")
-		}
-	}
-
-	for _, k := range numKeys {
-		v, ok := c[k]
-		if !ok {
-			continue
-		}
-
-		n, err := strconv.Atoi(v)
-		if err != nil || n < 0 {
-			return isSysErr, fmt.Errorf("invalid %s: %s", k, v)
-		}
-
 		if (k == common.EmailPort ||
 			k == common.MySQLPort) && n > 65535 {
-			return isSysErr, fmt.Errorf("invalid %s: %s", k, v)
+			return false, fmt.Errorf("invalid %s: %d", k, n)
 		}
 	}
 
-	if crt, ok := c[common.ProjectCreationRestriction]; ok &&
+	if crt, ok := strMap[common.ProjectCreationRestriction]; ok &&
 		crt != common.ProCrtRestrEveryone &&
 		crt != common.ProCrtRestrAdmOnly {
-		return isSysErr, fmt.Errorf("invalid %s, should be %s or %s",
+		return false, fmt.Errorf("invalid %s, should be %s or %s",
 			common.ProjectCreationRestriction,
 			common.ProCrtRestrAdmOnly,
 			common.ProCrtRestrEveryone)
 	}
-
+	return false, nil
-	return isSysErr, nil
-}
-
-//convert map[string]string to map[string]interface{}
-func convertForPut(m map[string]string) (map[string]interface{}, error) {
-	cfg := map[string]interface{}{}
-
-	for k, v := range m {
-		cfg[k] = v
-	}
-
-	for _, k := range numKeys {
-		if _, ok := cfg[k]; !ok {
-			continue
-		}
-
-		v, err := strconv.Atoi(cfg[k].(string))
-		if err != nil {
-			return nil, err
-		}
-		cfg[k] = v
-	}
-
-	for _, k := range boolKeys {
-		if _, ok := cfg[k]; !ok {
-			continue
-		}
-
-		cfg[k] = cfg[k] == "1"
-	}
-
-	return cfg, nil
 }
 
 // delete sensitive attrs and add editable field to every attr
@@ -347,6 +330,9 @@ func convertForGet(cfg map[string]interface{}) (map[string]*value, error) {
 		}
 	}
 
+	if _, ok := cfg[common.ScanAllPolicy]; !ok {
+		cfg[common.ScanAllPolicy] = models.DefaultScanAllPolicy
+	}
 	for k, v := range cfg {
 		result[k] = &value{
 			Value:    v,

src/ui/api/config_test.go

@@ -60,8 +60,8 @@ func TestPutConfig(t *testing.T) {
 	assert := assert.New(t)
 	apiTest := newHarborAPI()
 
-	cfg := map[string]string{
+	cfg := map[string]interface{}{
-		common.VerifyRemoteCert: "0",
+		common.VerifyRemoteCert: false,
 	}
 
 	code, err := apiTest.PutConfig(*admin, cfg)

src/ui/api/harborapi_test.go

@@ -1005,7 +1005,7 @@ func (a testapi) GetConfig(authInfo usrInfo) (int, map[string]*value, error) {
 	return code, cfg, err
 }
 
-func (a testapi) PutConfig(authInfo usrInfo, cfg map[string]string) (int, error) {
+func (a testapi) PutConfig(authInfo usrInfo, cfg map[string]interface{}) (int, error) {
 	_sling := sling.New().Base(a.basePath).Put("/api/configurations").BodyJSON(cfg)
 
 	code, _, err := request(_sling, jsonAcceptHeader, authInfo)

src/ui/config/config.go

@@ -16,6 +16,7 @@ package config
 
 import (
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"os"
@@ -355,7 +356,7 @@ func ClairEndpoint() string {
 func AdmiralEndpoint() string {
 	cfg, err := mg.Get()
 	if err != nil {
-		log.Errorf("Failed to get configuration, will return empty string as admiral's endpoint")
+		log.Errorf("Failed to get configuration, will return empty string as admiral's endpoint, error: %v", err)
 
 		return ""
 	}
@@ -365,6 +366,30 @@ func AdmiralEndpoint() string {
 	return cfg[common.AdmiralEndpoint].(string)
 }
 
+// ScanAllPolicy returns the policy which controls the scan all.
+func ScanAllPolicy() models.ScanAllPolicy {
+	var res models.ScanAllPolicy
+	cfg, err := mg.Get()
+	if err != nil {
+		log.Errorf("Failed to get configuration, will return default scan all policy, error: %v", err)
+		return models.DefaultScanAllPolicy
+	}
+	v, ok := cfg[common.ScanAllPolicy]
+	if !ok {
+		return models.DefaultScanAllPolicy
+	}
+	b, err := json.Marshal(v)
+	if err != nil {
+		log.Errorf("Failed to Marshal the value in configuration for Scan All policy, error: %v, returning the default policy", err)
+		return models.DefaultScanAllPolicy
+	}
+	if err := json.Unmarshal(b, &res); err != nil {
+		log.Errorf("Failed to unmarshal the value in configuration for Scan All policy, error: %v, returning the default policy", err)
+		return models.DefaultScanAllPolicy
+	}
+	return res
+}
+
 // WithAdmiral returns a bool to indicate if Harbor's deployed with admiral.
 func WithAdmiral() bool {
 	return len(AdmiralEndpoint()) > 0

src/ui/config/config_test.go

@@ -155,4 +155,9 @@ func TestConfig(t *testing.T) {
 	if mode != "db_auth" {
 		t.Errorf("unexpected mode: %s != %s", mode, "db_auth")
 	}
+
+	if s := ScanAllPolicy(); s.Type != "daily" {
+		t.Errorf("unexpected scan all policy %v", s)
+	}
+
 }

src/ui/proxy/interceptor_test.go

@@ -3,6 +3,7 @@ package proxy
 import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/vmware/harbor/src/adminserver/client"
 	"github.com/vmware/harbor/src/common"
 	"github.com/vmware/harbor/src/common/models"
 	notarytest "github.com/vmware/harbor/src/common/utils/notary/test"
@@ -19,6 +20,7 @@ import (
 var endpoint = "10.117.4.142"
 var notaryServer *httptest.Server
 var adminServer *httptest.Server
+var adminserverClient client.Client
 
 var admiralEndpoint = "http://127.0.0.1:8282"
 var token = ""
@@ -43,6 +45,7 @@ func TestMain(m *testing.M) {
 	if err := config.Init(); err != nil {
 		panic(err)
 	}
+	adminserverClient = client.NewClient(adminServer.URL, nil)
 	result := m.Run()
 	if result != 0 {
 		os.Exit(result)
@@ -95,19 +98,46 @@ func TestEnvPolicyChecker(t *testing.T) {
 	if err := os.Setenv("PROJECT_CONTENT_TRUST", "1"); err != nil {
 		t.Fatalf("Failed to set env variable: %v", err)
 	}
+	if err2 := os.Setenv("PROJECT_VULNERABLE", "1"); err2 != nil {
+		t.Fatalf("Failed to set env variable: %v", err2)
+	}
+	if err3 := os.Setenv("PROJECT_SEVERITY", "negligible"); err3 != nil {
+		t.Fatalf("Failed to set env variable: %v", err3)
+	}
 	contentTrustFlag := getPolicyChecker().contentTrustEnabled("whatever")
-	vulFlag := getPolicyChecker().vulnerableEnabled("whatever")
+	vulFlag, sev := getPolicyChecker().vulnerablePolicy("whatever")
 	assert.True(contentTrustFlag)
-	assert.False(vulFlag)
+	assert.True(vulFlag)
+	assert.Equal(sev, models.SevNone)
 }
 
 func TestPMSPolicyChecker(t *testing.T) {
+	var defaultConfigAdmiral = map[string]interface{}{
+		common.ExtEndpoint:     "https://" + endpoint,
+		common.WithNotary:      true,
+		common.CfgExpiration:   5,
+		common.AdmiralEndpoint: admiralEndpoint,
+	}
+	adminServer, err := utilstest.NewAdminserver(defaultConfigAdmiral)
+	if err != nil {
+		panic(err)
+	}
+	defer adminServer.Close()
+	if err := os.Setenv("ADMIN_SERVER_URL", adminServer.URL); err != nil {
+		panic(err)
+	}
+	if err := config.Init(); err != nil {
+		panic(err)
+	}
+
 	pm := pms.NewProjectManager(http.DefaultClient,
 		admiralEndpoint, nil)
-	name := "project_for_test_get_true"
+	name := "project_for_test_get_sev_low"
 	id, err := pm.Create(&models.Project{
-		Name:               name,
+		Name:                                       name,
-		EnableContentTrust: true,
+		EnableContentTrust:                         true,
+		PreventVulnerableImagesFromRunning:         false,
+		PreventVulnerableImagesFromRunningSeverity: "low",
 	})
 	require.Nil(t, err)
 	defer func(id int64) {
@@ -118,29 +148,27 @@ func TestPMSPolicyChecker(t *testing.T) {
 	project, err := pm.Get(id)
 	assert.Nil(t, err)
 	assert.Equal(t, id, project.ProjectID)
-	server, err2 := utilstest.NewAdminserver(nil)
+
-	if err2 != nil {
+	contentTrustFlag := getPolicyChecker().contentTrustEnabled("project_for_test_get_sev_low")
-		t.Fatalf("failed to create a mock admin server: %v", err2)
-	}
-	defer server.Close()
-	contentTrustFlag := getPolicyChecker().contentTrustEnabled("project_for_test_get_true")
 	assert.True(t, contentTrustFlag)
+	projectVulnerableEnabled, projectVulnerableSeverity := getPolicyChecker().vulnerablePolicy("project_for_test_get_sev_low")
+	assert.False(t, projectVulnerableEnabled)
+	assert.Equal(t, projectVulnerableSeverity, models.SevLow)
 }
 
 func TestMatchNotaryDigest(t *testing.T) {
 	assert := assert.New(t)
 	//The data from common/utils/notary/helper_test.go
-	img1 := imageInfo{"notary-demo/busybox", "1.0", "notary-demo"}
+	img1 := imageInfo{"notary-demo/busybox", "1.0", "notary-demo", "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"}
-	img2 := imageInfo{"notary-demo/busybox", "2.0", "notary-demo"}
+	img2 := imageInfo{"notary-demo/busybox", "2.0", "notary-demo", "sha256:12345678"}
-	res1, err := matchNotaryDigest(img1, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7")
+
+	res1, err := matchNotaryDigest(img1)
 	assert.Nil(err, "Unexpected error: %v, image: %#v", err, img1)
 	assert.True(res1)
-	res2, err := matchNotaryDigest(img1, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a8")
+
-	assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img1)
+	res2, err := matchNotaryDigest(img2)
+	assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img2)
 	assert.False(res2)
-	res3, err := matchNotaryDigest(img2, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7")
-	assert.Nil(err, "Unexpected error: %v, image: %#v", err, img2)
-	assert.False(res3)
 }
 
 func TestCopyResp(t *testing.T) {

src/ui/proxy/interceptors.go

@@ -1,9 +1,12 @@
 package proxy
 
 import (
-	//	"github.com/vmware/harbor/src/ui/api"
+	"github.com/vmware/harbor/src/common/dao"
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils/clair"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/common/utils/notary"
+	//	"github.com/vmware/harbor/src/ui/api"
 	"github.com/vmware/harbor/src/ui/config"
 	"github.com/vmware/harbor/src/ui/projectmanager"
 
@@ -25,6 +28,9 @@ const (
 	tokenUsername = "admin"
 )
 
+// Record the docker deamon raw response.
+var rec *httptest.ResponseRecorder
+
 // NotaryEndpoint , exported for testing.
 var NotaryEndpoint = config.InternalNotaryEndpoint()
 
@@ -50,8 +56,8 @@ func MatchPullManifest(req *http.Request) (bool, string, string) {
 type policyChecker interface {
 	// contentTrustEnabled returns whether a project has enabled content trust.
 	contentTrustEnabled(name string) bool
-	// vulnerableEnabled  returns whether a project has enabled content trust.
+	// vulnerablePolicy  returns whether a project has enabled vulnerable, and the project's severity.
-	vulnerableEnabled(name string) bool
+	vulnerablePolicy(name string) (bool, models.Severity)
 }
 
 //For testing
@@ -60,9 +66,8 @@ type envPolicyChecker struct{}
 func (ec envPolicyChecker) contentTrustEnabled(name string) bool {
 	return os.Getenv("PROJECT_CONTENT_TRUST") == "1"
 }
-func (ec envPolicyChecker) vulnerableEnabled(name string) bool {
+func (ec envPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity) {
-	// TODO: May need get more information in vulnerable policies.
+	return os.Getenv("PROJECT_VULNERABLE") == "1", clair.ParseClairSev(os.Getenv("PROJECT_SEVERITY"))
-	return os.Getenv("PROJECT_VULNERABBLE") == "1"
 }
 
 type pmsPolicyChecker struct {
@@ -77,8 +82,13 @@ func (pc pmsPolicyChecker) contentTrustEnabled(name string) bool {
 	}
 	return project.EnableContentTrust
 }
-func (pc pmsPolicyChecker) vulnerableEnabled(name string) bool {
+func (pc pmsPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity) {
-	return true
+	project, err := pc.pm.Get(name)
+	if err != nil {
+		log.Errorf("Unexpected error when getting the project, error: %v", err)
+		return true, models.SevUnknown
+	}
+	return project.PreventVulnerableImagesFromRunning, clair.ParseClairSev(project.PreventVulnerableImagesFromRunningSeverity)
 }
 
 // newPMSPolicyChecker returns an instance of an pmsPolicyChecker
@@ -100,7 +110,7 @@ type imageInfo struct {
 	repository  string
 	tag         string
 	projectName string
-	//	digest      string
+	digest      string
 }
 
 type urlHandler struct {
@@ -119,38 +129,22 @@ func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 			http.Error(rw, fmt.Sprintf("Bad repository name: %s", repository), http.StatusBadRequest)
 			return
 		}
-		/*
+		rec = httptest.NewRecorder()
-			//Need to get digest of the image.
+		uh.next.ServeHTTP(rec, req)
-			endpoint, err := config.RegistryURL()
+		if rec.Result().StatusCode != http.StatusOK {
-			if err != nil {
+			copyResp(rec, rw)
-				log.Errorf("Error getting Registry URL: %v", err)
+			return
-				http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError)
+		}
-				return
+		digest := rec.Header().Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
-			}
-			rc, err := api.NewRepositoryClient(endpoint, false, username, repository, "repository", repository, "pull")
-			if err != nil {
-				log.Errorf("Error creating repository Client: %v", err)
-				http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError)
-				return
-			}
-			digest, exist, err := rc.ManifestExist(tag)
-			if err != nil {
-				log.Errorf("Failed to get digest for tag: %s, error: %v", tag, err)
-				http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError)
-				return
-			}
-		*/
-
 		img := imageInfo{
 			repository:  repository,
 			tag:         tag,
 			projectName: components[0],
+			digest:      digest,
 		}
 		log.Debugf("image info of the request: %#v", img)
-
 		ctx := context.WithValue(req.Context(), imageInfoCtxKey, img)
 		req = req.WithContext(ctx)
-
 	}
 	uh.next.ServeHTTP(rw, req)
 }
@@ -170,31 +164,70 @@ func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Reque
 		cth.next.ServeHTTP(rw, req)
 		return
 	}
-	//May need to update status code, let's use recorder
+	match, err := matchNotaryDigest(img)
-	rec := httptest.NewRecorder()
-	cth.next.ServeHTTP(rec, req)
-	if rec.Result().StatusCode != http.StatusOK {
-		copyResp(rec, rw)
-		return
-	}
-	log.Debugf("showing digest")
-	digest := rec.Header().Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
-	log.Debugf("digest: %s", digest)
-	match, err := matchNotaryDigest(img, digest)
 	if err != nil {
 		http.Error(rw, "Failed in communication with Notary please check the log", http.StatusInternalServerError)
 		return
 	}
-	if match {
+	if !match {
-		log.Debugf("Passing the response to outter responseWriter")
-		copyResp(rec, rw)
-	} else {
 		log.Debugf("digest mismatch, failing the response.")
 		http.Error(rw, "The image is not signed in Notary.", http.StatusPreconditionFailed)
+		return
 	}
+	cth.next.ServeHTTP(rw, req)
 }
 
-func matchNotaryDigest(img imageInfo, digest string) (bool, error) {
+type vulnerableHandler struct {
+	next http.Handler
+}
+
+func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
+	imgRaw := req.Context().Value(imageInfoCtxKey)
+	if imgRaw == nil || !config.WithClair() {
+		vh.next.ServeHTTP(rw, req)
+		return
+	}
+	img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo)
+	projectVulnerableEnabled, projectVulnerableSeverity := getPolicyChecker().vulnerablePolicy(img.projectName)
+	if !projectVulnerableEnabled {
+		vh.next.ServeHTTP(rw, req)
+		return
+	}
+	overview, err := dao.GetImgScanOverview(img.digest)
+	if err != nil {
+		log.Errorf("failed to get ImgScanOverview with repo: %s, tag: %s, digest: %s. Error: %v", img.repository, img.tag, img.digest, err)
+		http.Error(rw, "Failed to get ImgScanOverview.", http.StatusPreconditionFailed)
+		return
+	}
+	if overview == nil {
+		log.Debugf("cannot get the image scan overview info, failing the response.")
+		http.Error(rw, "Cannot get the image scan overview info.", http.StatusPreconditionFailed)
+		return
+	}
+	imageSev := overview.Sev
+	if imageSev > int(projectVulnerableSeverity) {
+		log.Debugf("the image severity is higher then project setting, failing the response.")
+		http.Error(rw, "The image scan result doesn't pass the project setting.", http.StatusPreconditionFailed)
+		return
+	}
+	vh.next.ServeHTTP(rw, req)
+}
+
+type funnelHandler struct {
+	next http.Handler
+}
+
+func (fu funnelHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
+	imgRaw := req.Context().Value(imageInfoCtxKey)
+	if imgRaw != nil {
+		log.Debugf("Return the original response as no the interceptor takes action.")
+		copyResp(rec, rw)
+		return
+	}
+	fu.next.ServeHTTP(rw, req)
+}
+
+func matchNotaryDigest(img imageInfo) (bool, error) {
 	targets, err := notary.GetInternalTargets(NotaryEndpoint, tokenUsername, img.repository)
 	if err != nil {
 		return false, err
@@ -206,7 +239,7 @@ func matchNotaryDigest(img imageInfo, digest string) (bool, error) {
 			if err != nil {
 				return false, err
 			}
-			return digest == d, nil
+			return img.digest == d, nil
 		}
 	}
 	log.Debugf("image: %#v, not found in notary", img)

src/ui/proxy/proxy.go

@@ -42,7 +42,7 @@ func Init(urls ...string) error {
 	}
 	Proxy = httputil.NewSingleHostReverseProxy(targetURL)
 	//TODO: add vulnerable interceptor.
-	handlers = handlerChain{head: urlHandler{next: contentTrustHandler{next: Proxy}}}
+	handlers = handlerChain{head: urlHandler{next: contentTrustHandler{next: vulnerableHandler{next: funnelHandler{next: Proxy}}}}}
 	return nil
 }