diff --git a/make/common/templates/clair/postgresql-init.d/README.md b/make/common/templates/clair/postgresql-init.d/README.md new file mode 100644 index 000000000..b786ea8c6 --- /dev/null +++ b/make/common/templates/clair/postgresql-init.d/README.md @@ -0,0 +1,7 @@ +This folder used to run some initial sql for clair if needed. + +Just put the sql file in this directory and then start the +clair . + +both .sql and .gz format supported + diff --git a/make/docker-compose.clair.yml b/make/docker-compose.clair.yml index e5eb31867..3634cbe24 100644 --- a/make/docker-compose.clair.yml +++ b/make/docker-compose.clair.yml @@ -24,6 +24,7 @@ services: env_file: ./common/config/clair/postgres_env volumes: + - ./common/config/clair/postgresql-init.d/:/docker-entrypoint-initdb.d - /data/clair-db:/var/lib/postgresql/data logging: driver: "syslog" diff --git a/make/prepare b/make/prepare index 55144b897..42f4af71c 100755 --- a/make/prepare +++ b/make/prepare @@ -373,6 +373,10 @@ if args.clair_mode: pg_password = "password" clair_temp_dir = os.path.join(templates_dir, "clair") clair_config_dir = prep_conf_dir(config_dir, "clair") + print("Copying offline data file for clair DB") + if os.path.exists(os.path.join(clair_config_dir, "postgresql-init.d")): + shutil.rmtree(os.path.join(clair_config_dir, "postgresql-init.d")) + shutil.copytree(os.path.join(clair_temp_dir, "postgresql-init.d"), os.path.join(clair_config_dir, "postgresql-init.d")) postgres_env = os.path.join(clair_config_dir, "postgres_env") render(os.path.join(clair_temp_dir, "postgres_env"), postgres_env, password = pg_password) clair_conf = os.path.join(clair_config_dir, "config.yaml") diff --git a/src/common/const.go b/src/common/const.go index ca3bf1a26..e2bd9c60d 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -20,9 +20,9 @@ const ( LDAPAuth = "ldap_auth" ProCrtRestrEveryone = "everyone" ProCrtRestrAdmOnly = "adminonly" - LDAPScopeBase = "1" - LDAPScopeOnelevel = "2" - LDAPScopeSubtree = "3" + LDAPScopeBase = 1 + LDAPScopeOnelevel = 2 + LDAPScopeSubtree = 3 RoleProjectAdmin = 1 RoleDeveloper = 2 @@ -65,4 +65,5 @@ const ( AdmiralEndpoint = "admiral_url" WithNotary = "with_notary" WithClair = "with_clair" + ScanAllPolicy = "scan_all_policy" ) diff --git a/src/common/models/scan_job.go b/src/common/models/scan_job.go index f821a6dc6..51fffab28 100644 --- a/src/common/models/scan_job.go +++ b/src/common/models/scan_job.go @@ -96,3 +96,28 @@ type VulnerabilityItem struct { Description string `json:"description"` Fixed string `json:"fixedVersion,omitempty"` } + +// ScanAllPolicy is represent the json request and object for scan all policy, the parm is het +type ScanAllPolicy struct { + Type string `json:"type"` + Parm map[string]interface{} `json:"parameter, omitempty"` +} + +const ( + // ScanAllNone "none" for not doing any scan all + ScanAllNone = "none" + // ScanAllDaily for doing scan all daily + ScanAllDaily = "daily" + // ScanAllOnRefresh for doing scan all when the Clair DB is refreshed. + ScanAllOnRefresh = "on_refresh" + // ScanAllDailyTime the key for parm of daily scan all policy. + ScanAllDailyTime = "daily_time" +) + +//DefaultScanAllPolicy ... +var DefaultScanAllPolicy = ScanAllPolicy{ + Type: ScanAllDaily, + Parm: map[string]interface{}{ + ScanAllDailyTime: 0, + }, +} diff --git a/src/common/utils/registry/auth/tokenauthorizer.go b/src/common/utils/registry/auth/tokenauthorizer.go index b1766e5e0..118aa3444 100644 --- a/src/common/utils/registry/auth/tokenauthorizer.go +++ b/src/common/utils/registry/auth/tokenauthorizer.go @@ -15,20 +15,13 @@ package auth import ( - "encoding/json" "fmt" - "io/ioutil" "net/http" - "net/url" "strings" "sync" "time" - //"github.com/vmware/harbor/src/common/config" - "github.com/vmware/harbor/src/common/models" - "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/common/utils/registry" - registry_error "github.com/vmware/harbor/src/common/utils/error" token_util "github.com/vmware/harbor/src/ui/service/token" ) @@ -36,13 +29,14 @@ const ( latency int = 10 //second, the network latency when token is received ) -type scope struct { +// Scope ... +type Scope struct { Type string Name string Actions []string } -func (s *scope) string() string { +func (s *Scope) string() string { return fmt.Sprintf("%s:%s:%s", s.Type, s.Name, strings.Join(s.Actions, ",")) } @@ -50,7 +44,7 @@ type tokenGenerator func(realm, service string, scopes []string) (token string, // Implements interface Authorizer type tokenAuthorizer struct { - scope *scope + scope *Scope tg tokenGenerator cache string // cached token expiresAt *time.Time // The UTC standard time at when the token will expire @@ -64,13 +58,13 @@ func (t *tokenAuthorizer) Scheme() string { // AuthorizeRequest will add authorization header which contains a token before the request is sent func (t *tokenAuthorizer) Authorize(req *http.Request, params map[string]string) error { - var scopes []*scope + var scopes []*Scope var token string hasFrom := false from := req.URL.Query().Get("from") if len(from) != 0 { - s := &scope{ + s := &Scope{ Type: "repository", Name: from, Actions: []string{"pull"}, @@ -154,7 +148,7 @@ func NewStandardTokenAuthorizer(credential Credential, insecure bool, } if len(scopeType) != 0 || len(scopeName) != 0 { - authorizer.scope = &scope{ + authorizer.scope = &Scope{ Type: scopeType, Name: scopeName, Actions: scopeActions, @@ -166,66 +160,21 @@ func NewStandardTokenAuthorizer(credential Credential, insecure bool, return authorizer } -func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) { +func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes []string) (string, int, *time.Time, error) { realm = s.tokenURL(realm) + tk, err := getToken(s.client, s.credential, realm, + service, scopes) - u, err := url.Parse(realm) + if len(tk.IssuedAt) == 0 { + return tk.Token, tk.ExpiresIn, nil, nil + } + + issuedAt, err := time.Parse(time.RFC3339, tk.IssuedAt) if err != nil { - return - } - q := u.Query() - q.Add("service", service) - for _, scope := range scopes { - q.Add("scope", scope) - } - u.RawQuery = q.Encode() - r, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return + return "", 0, nil, err } - if s.credential != nil { - s.credential.AddAuthorization(r) - } - - resp, err := s.client.Do(r) - if err != nil { - return - } - - defer resp.Body.Close() - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return - } - if resp.StatusCode != http.StatusOK { - err = ®istry_error.Error{ - StatusCode: resp.StatusCode, - Detail: string(b), - } - return - } - - tk := models.Token{} - if err = json.Unmarshal(b, &tk); err != nil { - return - } - - token = tk.Token - - expiresIn = tk.ExpiresIn - - if len(tk.IssuedAt) != 0 { - t, err := time.Parse(time.RFC3339, tk.IssuedAt) - if err != nil { - log.Errorf("error occurred while parsing issued_at: %v", err) - err = nil - } else { - issuedAt = &t - } - } - - return + return tk.Token, tk.ExpiresIn, &issuedAt, nil } // when the registry client is used inside Harbor, the token request @@ -267,7 +216,7 @@ func newUsernameTokenAuthorizer(notary bool, username, scopeType, scopeName stri username: username, } - authorizer.scope = &scope{ + authorizer.scope = &Scope{ Type: scopeType, Name: scopeName, Actions: scopeActions, diff --git a/src/common/utils/registry/auth/util.go b/src/common/utils/registry/auth/util.go new file mode 100644 index 000000000..719418294 --- /dev/null +++ b/src/common/utils/registry/auth/util.go @@ -0,0 +1,93 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "net/url" + + "github.com/vmware/harbor/src/common/models" + + registry_error "github.com/vmware/harbor/src/common/utils/error" + "github.com/vmware/harbor/src/common/utils/registry" +) + +const ( + service = "harbor-registry" +) + +// GetToken requests a token against the endpoint using credetial provided +func GetToken(endpoint string, insecure bool, credential Credential, + scopes []*Scope) (*models.Token, error) { + client := &http.Client{ + Transport: registry.GetHTTPTransport(insecure), + } + + scopesStr := []string{} + for _, scope := range scopes { + scopesStr = append(scopesStr, scope.string()) + } + + return getToken(client, credential, endpoint, service, scopesStr) +} + +func getToken(client *http.Client, credential Credential, realm, service string, + scopes []string) (*models.Token, error) { + u, err := url.Parse(realm) + if err != nil { + return nil, err + } + query := u.Query() + query.Add("service", service) + for _, scope := range scopes { + query.Add("scope", scope) + } + u.RawQuery = query.Encode() + + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + + if credential != nil { + credential.AddAuthorization(req) + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, ®istry_error.Error{ + StatusCode: resp.StatusCode, + Detail: string(data), + } + } + + token := &models.Token{} + if err = json.Unmarshal(data, token); err != nil { + return nil, err + } + + return token, nil +} diff --git a/src/jobservice/utils/utils.go b/src/jobservice/utils/utils.go index 30546e777..ce6b775f5 100644 --- a/src/jobservice/utils/utils.go +++ b/src/jobservice/utils/utils.go @@ -15,11 +15,8 @@ package utils import ( - "encoding/json" "fmt" - "io/ioutil" "net/http" - "net/url" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/registry" @@ -64,39 +61,18 @@ func BuildBlobURL(endpoint, repository, digest string) string { return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repository, digest) } -//GetTokenForRepo is a temp solution for job handler to get a token for clair. -//TODO: Get rid of it when it can get a token from repository client. +//GetTokenForRepo is used for job handler to get a token for clair. func GetTokenForRepo(repository string) (string, error) { - u, err := url.Parse(config.InternalTokenServiceEndpoint()) - if err != nil { - return "", err - } - q := u.Query() - q.Add("service", "harbor-registry") - q.Add("scope", fmt.Sprintf("repository:%s:pull", repository)) - u.RawQuery = q.Encode() - r, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return "", err - } c := &http.Cookie{Name: models.UISecretCookie, Value: config.JobserviceSecret()} - r.AddCookie(c) - client := &http.Client{} - resp, err := client.Do(r) + credentail := auth.NewCookieCredential(c) + token, err := auth.GetToken(config.InternalTokenServiceEndpoint(), true, credentail, []*auth.Scope{&auth.Scope{ + Type: "repository", + Name: repository, + Actions: []string{"pull"}, + }}) if err != nil { return "", err } - defer resp.Body.Close() - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("Unexpected response from token service, code: %d, %s", resp.StatusCode, string(b)) - } - tk := models.Token{} - if err := json.Unmarshal(b, &tk); err != nil { - return "", err - } - return tk.Token, nil + + return token.Token, nil } diff --git a/src/ui/api/config.go b/src/ui/api/config.go index 42bac0e54..a1b6f8b1b 100644 --- a/src/ui/api/config.go +++ b/src/ui/api/config.go @@ -17,10 +17,11 @@ package api import ( "fmt" "net/http" - "strconv" + "reflect" "github.com/vmware/harbor/src/common" "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/config" ) @@ -62,6 +63,34 @@ var ( common.CfgExpiration, common.JobLogDir, common.AdminInitialPassword, + common.ScanAllPolicy, + } + + stringKeys = []string{ + common.ExtEndpoint, + common.AUTHMode, + common.DatabaseType, + common.MySQLHost, + common.MySQLUsername, + common.MySQLPassword, + common.MySQLDatabase, + common.SQLiteFile, + common.LDAPURL, + common.LDAPSearchDN, + common.LDAPSearchPwd, + common.LDAPBaseDN, + common.LDAPUID, + common.LDAPFilter, + common.TokenServiceURL, + common.RegistryURL, + common.EmailHost, + common.EmailUsername, + common.EmailPassword, + common.EmailFrom, + common.EmailIdentity, + common.ProjectCreationRestriction, + common.JobLogDir, + common.AdminInitialPassword, } numKeys = []string{ @@ -131,10 +160,10 @@ func (c *ConfigAPI) Get() { // Put updates configurations func (c *ConfigAPI) Put() { - m := map[string]string{} + m := map[string]interface{}{} c.DecodeJSONReq(&m) - cfg := map[string]string{} + cfg := map[string]interface{}{} for _, k := range validKeys { if v, ok := m[k]; ok { cfg[k] = v @@ -152,35 +181,7 @@ func (c *ConfigAPI) Put() { c.CustomAbort(http.StatusBadRequest, err.Error()) } - if value, ok := cfg[common.AUTHMode]; ok { - mode, err := config.AuthMode() - if err != nil { - log.Errorf("failed to get auth mode: %v", err) - c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - if mode != value { - flag, err := authModeCanBeModified() - if err != nil { - log.Errorf("failed to determine whether auth mode can be modified: %v", err) - c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - if !flag { - c.CustomAbort(http.StatusBadRequest, - fmt.Sprintf("%s can not be modified as new users have been inserted into database", - common.AUTHMode)) - } - } - } - - result, err := convertForPut(cfg) - if err != nil { - log.Errorf("failed to convert configurations: %v", err) - c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) - } - - if err := config.Upload(result); err != nil { + if err := config.Upload(cfg); err != nil { log.Errorf("failed to upload configurations: %v", err) c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } @@ -199,18 +200,53 @@ func (c *ConfigAPI) Reset() { } } -func validateCfg(c map[string]string) (bool, error) { - isSysErr := false +func validateCfg(c map[string]interface{}) (bool, error) { + strMap := map[string]string{} + for _, k := range stringKeys { + if _, ok := c[k]; !ok { + continue + } + if _, ok := c[k].(string); !ok { + return false, fmt.Errorf("Invalid value type, expected string, key: %s, value: %v, type: %v", k, c[k], reflect.TypeOf(c[k])) + } + strMap[k] = c[k].(string) + } + numMap := map[string]int{} + for _, k := range numKeys { + if _, ok := c[k]; !ok { + continue + } + if _, ok := c[k].(float64); !ok { + return false, fmt.Errorf("Invalid value type, expected float64, key: %s, value: %v, type: %v", k, c[k], reflect.TypeOf(c[k])) + } + numMap[k] = int(c[k].(float64)) + } + boolMap := map[string]bool{} + for _, k := range boolKeys { + if _, ok := c[k]; !ok { + continue + } + if _, ok := c[k].(bool); !ok { + return false, fmt.Errorf("Invalid value type, expected bool, key: %s, value: %v, type: %v", k, c[k], reflect.TypeOf(c[k])) + } + boolMap[k] = c[k].(bool) + } mode, err := config.AuthMode() if err != nil { - isSysErr = true - return isSysErr, err + return true, err } - if value, ok := c[common.AUTHMode]; ok { + if value, ok := strMap[common.AUTHMode]; ok { if value != common.DBAuth && value != common.LDAPAuth { - return isSysErr, fmt.Errorf("invalid %s, shoud be %s or %s", common.AUTHMode, common.DBAuth, common.LDAPAuth) + return false, fmt.Errorf("invalid %s, shoud be %s or %s", common.AUTHMode, common.DBAuth, common.LDAPAuth) + } + flag, err := authModeCanBeModified() + if err != nil { + return true, err + } + if mode != value && !flag { + return false, fmt.Errorf("%s can not be modified as new users have been inserted into database", common.AUTHMode) } mode = value } @@ -218,123 +254,70 @@ func validateCfg(c map[string]string) (bool, error) { if mode == common.LDAPAuth { ldap, err := config.LDAP() if err != nil { - isSysErr = true - return isSysErr, err + return true, err } if len(ldap.URL) == 0 { - if _, ok := c[common.LDAPURL]; !ok { - return isSysErr, fmt.Errorf("%s is missing", common.LDAPURL) + if _, ok := strMap[common.LDAPURL]; !ok { + return false, fmt.Errorf("%s is missing", common.LDAPURL) } } if len(ldap.BaseDN) == 0 { - if _, ok := c[common.LDAPBaseDN]; !ok { - return isSysErr, fmt.Errorf("%s is missing", common.LDAPBaseDN) + if _, ok := strMap[common.LDAPBaseDN]; !ok { + return false, fmt.Errorf("%s is missing", common.LDAPBaseDN) } } if len(ldap.UID) == 0 { - if _, ok := c[common.LDAPUID]; !ok { - return isSysErr, fmt.Errorf("%s is missing", common.LDAPUID) + if _, ok := strMap[common.LDAPUID]; !ok { + return false, fmt.Errorf("%s is missing", common.LDAPUID) } } if ldap.Scope == 0 { - if _, ok := c[common.LDAPScope]; !ok { - return isSysErr, fmt.Errorf("%s is missing", common.LDAPScope) + if _, ok := numMap[common.LDAPScope]; !ok { + return false, fmt.Errorf("%s is missing", common.LDAPScope) } } } - if ldapURL, ok := c[common.LDAPURL]; ok && len(ldapURL) == 0 { - return isSysErr, fmt.Errorf("%s is empty", common.LDAPURL) + if ldapURL, ok := strMap[common.LDAPURL]; ok && len(ldapURL) == 0 { + return false, fmt.Errorf("%s is empty", common.LDAPURL) } - if baseDN, ok := c[common.LDAPBaseDN]; ok && len(baseDN) == 0 { - return isSysErr, fmt.Errorf("%s is empty", common.LDAPBaseDN) + if baseDN, ok := strMap[common.LDAPBaseDN]; ok && len(baseDN) == 0 { + return false, fmt.Errorf("%s is empty", common.LDAPBaseDN) } - if uID, ok := c[common.LDAPUID]; ok && len(uID) == 0 { - return isSysErr, fmt.Errorf("%s is empty", common.LDAPUID) + if uID, ok := strMap[common.LDAPUID]; ok && len(uID) == 0 { + return false, fmt.Errorf("%s is empty", common.LDAPUID) } - if scope, ok := c[common.LDAPScope]; ok && + if scope, ok := numMap[common.LDAPScope]; ok && scope != common.LDAPScopeBase && scope != common.LDAPScopeOnelevel && scope != common.LDAPScopeSubtree { - return isSysErr, fmt.Errorf("invalid %s, should be %s, %s or %s", + return false, fmt.Errorf("invalid %s, should be %s, %s or %s", common.LDAPScope, common.LDAPScopeBase, common.LDAPScopeOnelevel, common.LDAPScopeSubtree) } - - for _, k := range boolKeys { - v, ok := c[k] - if !ok { - continue + for k, n := range numMap { + if n < 0 { + return false, fmt.Errorf("invalid %s: %d", k, n) } - - if v != "0" && v != "1" { - return isSysErr, fmt.Errorf("%s should be %s or %s", - k, "0", "1") - } - } - - for _, k := range numKeys { - v, ok := c[k] - if !ok { - continue - } - - n, err := strconv.Atoi(v) - if err != nil || n < 0 { - return isSysErr, fmt.Errorf("invalid %s: %s", k, v) - } - if (k == common.EmailPort || k == common.MySQLPort) && n > 65535 { - return isSysErr, fmt.Errorf("invalid %s: %s", k, v) + return false, fmt.Errorf("invalid %s: %d", k, n) } } - if crt, ok := c[common.ProjectCreationRestriction]; ok && + if crt, ok := strMap[common.ProjectCreationRestriction]; ok && crt != common.ProCrtRestrEveryone && crt != common.ProCrtRestrAdmOnly { - return isSysErr, fmt.Errorf("invalid %s, should be %s or %s", + return false, fmt.Errorf("invalid %s, should be %s or %s", common.ProjectCreationRestriction, common.ProCrtRestrAdmOnly, common.ProCrtRestrEveryone) } - - return isSysErr, nil -} - -//convert map[string]string to map[string]interface{} -func convertForPut(m map[string]string) (map[string]interface{}, error) { - cfg := map[string]interface{}{} - - for k, v := range m { - cfg[k] = v - } - - for _, k := range numKeys { - if _, ok := cfg[k]; !ok { - continue - } - - v, err := strconv.Atoi(cfg[k].(string)) - if err != nil { - return nil, err - } - cfg[k] = v - } - - for _, k := range boolKeys { - if _, ok := cfg[k]; !ok { - continue - } - - cfg[k] = cfg[k] == "1" - } - - return cfg, nil + return false, nil } // delete sensitive attrs and add editable field to every attr @@ -347,6 +330,9 @@ func convertForGet(cfg map[string]interface{}) (map[string]*value, error) { } } + if _, ok := cfg[common.ScanAllPolicy]; !ok { + cfg[common.ScanAllPolicy] = models.DefaultScanAllPolicy + } for k, v := range cfg { result[k] = &value{ Value: v, diff --git a/src/ui/api/config_test.go b/src/ui/api/config_test.go index 1b83759ec..eeffdade8 100644 --- a/src/ui/api/config_test.go +++ b/src/ui/api/config_test.go @@ -60,8 +60,8 @@ func TestPutConfig(t *testing.T) { assert := assert.New(t) apiTest := newHarborAPI() - cfg := map[string]string{ - common.VerifyRemoteCert: "0", + cfg := map[string]interface{}{ + common.VerifyRemoteCert: false, } code, err := apiTest.PutConfig(*admin, cfg) diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index 3b580d9cc..d9f1d234a 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -1005,7 +1005,7 @@ func (a testapi) GetConfig(authInfo usrInfo) (int, map[string]*value, error) { return code, cfg, err } -func (a testapi) PutConfig(authInfo usrInfo, cfg map[string]string) (int, error) { +func (a testapi) PutConfig(authInfo usrInfo, cfg map[string]interface{}) (int, error) { _sling := sling.New().Base(a.basePath).Put("/api/configurations").BodyJSON(cfg) code, _, err := request(_sling, jsonAcceptHeader, authInfo) diff --git a/src/ui/config/config.go b/src/ui/config/config.go index de14f12f4..ca4a61e45 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -16,6 +16,7 @@ package config import ( "crypto/tls" + "encoding/json" "fmt" "net/http" "os" @@ -355,7 +356,7 @@ func ClairEndpoint() string { func AdmiralEndpoint() string { cfg, err := mg.Get() if err != nil { - log.Errorf("Failed to get configuration, will return empty string as admiral's endpoint") + log.Errorf("Failed to get configuration, will return empty string as admiral's endpoint, error: %v", err) return "" } @@ -365,6 +366,30 @@ func AdmiralEndpoint() string { return cfg[common.AdmiralEndpoint].(string) } +// ScanAllPolicy returns the policy which controls the scan all. +func ScanAllPolicy() models.ScanAllPolicy { + var res models.ScanAllPolicy + cfg, err := mg.Get() + if err != nil { + log.Errorf("Failed to get configuration, will return default scan all policy, error: %v", err) + return models.DefaultScanAllPolicy + } + v, ok := cfg[common.ScanAllPolicy] + if !ok { + return models.DefaultScanAllPolicy + } + b, err := json.Marshal(v) + if err != nil { + log.Errorf("Failed to Marshal the value in configuration for Scan All policy, error: %v, returning the default policy", err) + return models.DefaultScanAllPolicy + } + if err := json.Unmarshal(b, &res); err != nil { + log.Errorf("Failed to unmarshal the value in configuration for Scan All policy, error: %v, returning the default policy", err) + return models.DefaultScanAllPolicy + } + return res +} + // WithAdmiral returns a bool to indicate if Harbor's deployed with admiral. func WithAdmiral() bool { return len(AdmiralEndpoint()) > 0 diff --git a/src/ui/config/config_test.go b/src/ui/config/config_test.go index c7655a75e..c496b06ad 100644 --- a/src/ui/config/config_test.go +++ b/src/ui/config/config_test.go @@ -155,4 +155,9 @@ func TestConfig(t *testing.T) { if mode != "db_auth" { t.Errorf("unexpected mode: %s != %s", mode, "db_auth") } + + if s := ScanAllPolicy(); s.Type != "daily" { + t.Errorf("unexpected scan all policy %v", s) + } + } diff --git a/src/ui/proxy/interceptor_test.go b/src/ui/proxy/interceptor_test.go index bd739fb3a..0c84f9ba9 100644 --- a/src/ui/proxy/interceptor_test.go +++ b/src/ui/proxy/interceptor_test.go @@ -3,6 +3,7 @@ package proxy import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/vmware/harbor/src/adminserver/client" "github.com/vmware/harbor/src/common" "github.com/vmware/harbor/src/common/models" notarytest "github.com/vmware/harbor/src/common/utils/notary/test" @@ -19,6 +20,7 @@ import ( var endpoint = "10.117.4.142" var notaryServer *httptest.Server var adminServer *httptest.Server +var adminserverClient client.Client var admiralEndpoint = "http://127.0.0.1:8282" var token = "" @@ -43,6 +45,7 @@ func TestMain(m *testing.M) { if err := config.Init(); err != nil { panic(err) } + adminserverClient = client.NewClient(adminServer.URL, nil) result := m.Run() if result != 0 { os.Exit(result) @@ -95,19 +98,46 @@ func TestEnvPolicyChecker(t *testing.T) { if err := os.Setenv("PROJECT_CONTENT_TRUST", "1"); err != nil { t.Fatalf("Failed to set env variable: %v", err) } + if err2 := os.Setenv("PROJECT_VULNERABLE", "1"); err2 != nil { + t.Fatalf("Failed to set env variable: %v", err2) + } + if err3 := os.Setenv("PROJECT_SEVERITY", "negligible"); err3 != nil { + t.Fatalf("Failed to set env variable: %v", err3) + } contentTrustFlag := getPolicyChecker().contentTrustEnabled("whatever") - vulFlag := getPolicyChecker().vulnerableEnabled("whatever") + vulFlag, sev := getPolicyChecker().vulnerablePolicy("whatever") assert.True(contentTrustFlag) - assert.False(vulFlag) + assert.True(vulFlag) + assert.Equal(sev, models.SevNone) } func TestPMSPolicyChecker(t *testing.T) { + var defaultConfigAdmiral = map[string]interface{}{ + common.ExtEndpoint: "https://" + endpoint, + common.WithNotary: true, + common.CfgExpiration: 5, + common.AdmiralEndpoint: admiralEndpoint, + } + adminServer, err := utilstest.NewAdminserver(defaultConfigAdmiral) + if err != nil { + panic(err) + } + defer adminServer.Close() + if err := os.Setenv("ADMIN_SERVER_URL", adminServer.URL); err != nil { + panic(err) + } + if err := config.Init(); err != nil { + panic(err) + } + pm := pms.NewProjectManager(http.DefaultClient, admiralEndpoint, nil) - name := "project_for_test_get_true" + name := "project_for_test_get_sev_low" id, err := pm.Create(&models.Project{ - Name: name, - EnableContentTrust: true, + Name: name, + EnableContentTrust: true, + PreventVulnerableImagesFromRunning: false, + PreventVulnerableImagesFromRunningSeverity: "low", }) require.Nil(t, err) defer func(id int64) { @@ -118,29 +148,27 @@ func TestPMSPolicyChecker(t *testing.T) { project, err := pm.Get(id) assert.Nil(t, err) assert.Equal(t, id, project.ProjectID) - server, err2 := utilstest.NewAdminserver(nil) - if err2 != nil { - t.Fatalf("failed to create a mock admin server: %v", err2) - } - defer server.Close() - contentTrustFlag := getPolicyChecker().contentTrustEnabled("project_for_test_get_true") + + contentTrustFlag := getPolicyChecker().contentTrustEnabled("project_for_test_get_sev_low") assert.True(t, contentTrustFlag) + projectVulnerableEnabled, projectVulnerableSeverity := getPolicyChecker().vulnerablePolicy("project_for_test_get_sev_low") + assert.False(t, projectVulnerableEnabled) + assert.Equal(t, projectVulnerableSeverity, models.SevLow) } func TestMatchNotaryDigest(t *testing.T) { assert := assert.New(t) //The data from common/utils/notary/helper_test.go - img1 := imageInfo{"notary-demo/busybox", "1.0", "notary-demo"} - img2 := imageInfo{"notary-demo/busybox", "2.0", "notary-demo"} - res1, err := matchNotaryDigest(img1, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7") + img1 := imageInfo{"notary-demo/busybox", "1.0", "notary-demo", "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"} + img2 := imageInfo{"notary-demo/busybox", "2.0", "notary-demo", "sha256:12345678"} + + res1, err := matchNotaryDigest(img1) assert.Nil(err, "Unexpected error: %v, image: %#v", err, img1) assert.True(res1) - res2, err := matchNotaryDigest(img1, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a8") - assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img1) + + res2, err := matchNotaryDigest(img2) + assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img2) assert.False(res2) - res3, err := matchNotaryDigest(img2, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7") - assert.Nil(err, "Unexpected error: %v, image: %#v", err, img2) - assert.False(res3) } func TestCopyResp(t *testing.T) { diff --git a/src/ui/proxy/interceptors.go b/src/ui/proxy/interceptors.go index 49f4bc64c..af32a834f 100644 --- a/src/ui/proxy/interceptors.go +++ b/src/ui/proxy/interceptors.go @@ -1,9 +1,12 @@ package proxy import ( - // "github.com/vmware/harbor/src/ui/api" + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/clair" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/common/utils/notary" + // "github.com/vmware/harbor/src/ui/api" "github.com/vmware/harbor/src/ui/config" "github.com/vmware/harbor/src/ui/projectmanager" @@ -25,6 +28,9 @@ const ( tokenUsername = "admin" ) +// Record the docker deamon raw response. +var rec *httptest.ResponseRecorder + // NotaryEndpoint , exported for testing. var NotaryEndpoint = config.InternalNotaryEndpoint() @@ -50,8 +56,8 @@ func MatchPullManifest(req *http.Request) (bool, string, string) { type policyChecker interface { // contentTrustEnabled returns whether a project has enabled content trust. contentTrustEnabled(name string) bool - // vulnerableEnabled returns whether a project has enabled content trust. - vulnerableEnabled(name string) bool + // vulnerablePolicy returns whether a project has enabled vulnerable, and the project's severity. + vulnerablePolicy(name string) (bool, models.Severity) } //For testing @@ -60,9 +66,8 @@ type envPolicyChecker struct{} func (ec envPolicyChecker) contentTrustEnabled(name string) bool { return os.Getenv("PROJECT_CONTENT_TRUST") == "1" } -func (ec envPolicyChecker) vulnerableEnabled(name string) bool { - // TODO: May need get more information in vulnerable policies. - return os.Getenv("PROJECT_VULNERABBLE") == "1" +func (ec envPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity) { + return os.Getenv("PROJECT_VULNERABLE") == "1", clair.ParseClairSev(os.Getenv("PROJECT_SEVERITY")) } type pmsPolicyChecker struct { @@ -77,8 +82,13 @@ func (pc pmsPolicyChecker) contentTrustEnabled(name string) bool { } return project.EnableContentTrust } -func (pc pmsPolicyChecker) vulnerableEnabled(name string) bool { - return true +func (pc pmsPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity) { + project, err := pc.pm.Get(name) + if err != nil { + log.Errorf("Unexpected error when getting the project, error: %v", err) + return true, models.SevUnknown + } + return project.PreventVulnerableImagesFromRunning, clair.ParseClairSev(project.PreventVulnerableImagesFromRunningSeverity) } // newPMSPolicyChecker returns an instance of an pmsPolicyChecker @@ -100,7 +110,7 @@ type imageInfo struct { repository string tag string projectName string - // digest string + digest string } type urlHandler struct { @@ -119,38 +129,22 @@ func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { http.Error(rw, fmt.Sprintf("Bad repository name: %s", repository), http.StatusBadRequest) return } - /* - //Need to get digest of the image. - endpoint, err := config.RegistryURL() - if err != nil { - log.Errorf("Error getting Registry URL: %v", err) - http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError) - return - } - rc, err := api.NewRepositoryClient(endpoint, false, username, repository, "repository", repository, "pull") - if err != nil { - log.Errorf("Error creating repository Client: %v", err) - http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError) - return - } - digest, exist, err := rc.ManifestExist(tag) - if err != nil { - log.Errorf("Failed to get digest for tag: %s, error: %v", tag, err) - http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError) - return - } - */ - + rec = httptest.NewRecorder() + uh.next.ServeHTTP(rec, req) + if rec.Result().StatusCode != http.StatusOK { + copyResp(rec, rw) + return + } + digest := rec.Header().Get(http.CanonicalHeaderKey("Docker-Content-Digest")) img := imageInfo{ repository: repository, tag: tag, projectName: components[0], + digest: digest, } log.Debugf("image info of the request: %#v", img) - ctx := context.WithValue(req.Context(), imageInfoCtxKey, img) req = req.WithContext(ctx) - } uh.next.ServeHTTP(rw, req) } @@ -170,31 +164,70 @@ func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Reque cth.next.ServeHTTP(rw, req) return } - //May need to update status code, let's use recorder - rec := httptest.NewRecorder() - cth.next.ServeHTTP(rec, req) - if rec.Result().StatusCode != http.StatusOK { - copyResp(rec, rw) - return - } - log.Debugf("showing digest") - digest := rec.Header().Get(http.CanonicalHeaderKey("Docker-Content-Digest")) - log.Debugf("digest: %s", digest) - match, err := matchNotaryDigest(img, digest) + match, err := matchNotaryDigest(img) if err != nil { http.Error(rw, "Failed in communication with Notary please check the log", http.StatusInternalServerError) return } - if match { - log.Debugf("Passing the response to outter responseWriter") - copyResp(rec, rw) - } else { + if !match { log.Debugf("digest mismatch, failing the response.") http.Error(rw, "The image is not signed in Notary.", http.StatusPreconditionFailed) + return } + cth.next.ServeHTTP(rw, req) } -func matchNotaryDigest(img imageInfo, digest string) (bool, error) { +type vulnerableHandler struct { + next http.Handler +} + +func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + imgRaw := req.Context().Value(imageInfoCtxKey) + if imgRaw == nil || !config.WithClair() { + vh.next.ServeHTTP(rw, req) + return + } + img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo) + projectVulnerableEnabled, projectVulnerableSeverity := getPolicyChecker().vulnerablePolicy(img.projectName) + if !projectVulnerableEnabled { + vh.next.ServeHTTP(rw, req) + return + } + overview, err := dao.GetImgScanOverview(img.digest) + if err != nil { + log.Errorf("failed to get ImgScanOverview with repo: %s, tag: %s, digest: %s. Error: %v", img.repository, img.tag, img.digest, err) + http.Error(rw, "Failed to get ImgScanOverview.", http.StatusPreconditionFailed) + return + } + if overview == nil { + log.Debugf("cannot get the image scan overview info, failing the response.") + http.Error(rw, "Cannot get the image scan overview info.", http.StatusPreconditionFailed) + return + } + imageSev := overview.Sev + if imageSev > int(projectVulnerableSeverity) { + log.Debugf("the image severity is higher then project setting, failing the response.") + http.Error(rw, "The image scan result doesn't pass the project setting.", http.StatusPreconditionFailed) + return + } + vh.next.ServeHTTP(rw, req) +} + +type funnelHandler struct { + next http.Handler +} + +func (fu funnelHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + imgRaw := req.Context().Value(imageInfoCtxKey) + if imgRaw != nil { + log.Debugf("Return the original response as no the interceptor takes action.") + copyResp(rec, rw) + return + } + fu.next.ServeHTTP(rw, req) +} + +func matchNotaryDigest(img imageInfo) (bool, error) { targets, err := notary.GetInternalTargets(NotaryEndpoint, tokenUsername, img.repository) if err != nil { return false, err @@ -206,7 +239,7 @@ func matchNotaryDigest(img imageInfo, digest string) (bool, error) { if err != nil { return false, err } - return digest == d, nil + return img.digest == d, nil } } log.Debugf("image: %#v, not found in notary", img) diff --git a/src/ui/proxy/proxy.go b/src/ui/proxy/proxy.go index b580d7aff..be8d5b728 100644 --- a/src/ui/proxy/proxy.go +++ b/src/ui/proxy/proxy.go @@ -42,7 +42,7 @@ func Init(urls ...string) error { } Proxy = httputil.NewSingleHostReverseProxy(targetURL) //TODO: add vulnerable interceptor. - handlers = handlerChain{head: urlHandler{next: contentTrustHandler{next: Proxy}}} + handlers = handlerChain{head: urlHandler{next: contentTrustHandler{next: vulnerableHandler{next: funnelHandler{next: Proxy}}}}} return nil }