Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-07-03 18:25:01 +02:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into 170628_getpm

Browse Source 
Conflicts:
	src/ui/config/config.go
	src/ui/proxy/interceptor_test.go
This commit is contained in:
Wenkai Yin 2017-06-30 00:14:45 +08:00
commit 84b443e2b2
16 changed files with 427 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
make/common/templates/clair/postgresql-init.d/README.md Normal file
View File

@ -0,0 +1,7 @@
This folder used to run some initial sql for clair if needed.
Just put the sql file in this directory and then start the
clair .
both .sql and .gz format supported

1
make/docker-compose.clair.yml
View File

@ -24,6 +24,7 @@ services:
env_file:
./common/config/clair/postgres_env
volumes:
- ./common/config/clair/postgresql-init.d/:/docker-entrypoint-initdb.d
- /data/clair-db:/var/lib/postgresql/data
logging:
driver: "syslog"

4
make/prepare
View File

@ -373,6 +373,10 @@ if args.clair_mode:
pg_password = "password"
clair_temp_dir = os.path.join(templates_dir, "clair")
clair_config_dir = prep_conf_dir(config_dir, "clair")
print("Copying offline data file for clair DB")
if os.path.exists(os.path.join(clair_config_dir, "postgresql-init.d")):
shutil.rmtree(os.path.join(clair_config_dir, "postgresql-init.d"))
shutil.copytree(os.path.join(clair_temp_dir, "postgresql-init.d"), os.path.join(clair_config_dir, "postgresql-init.d"))
postgres_env = os.path.join(clair_config_dir, "postgres_env")
render(os.path.join(clair_temp_dir, "postgres_env"), postgres_env, password = pg_password)
clair_conf = os.path.join(clair_config_dir, "config.yaml")

7
src/common/const.go
View File

@ -20,9 +20,9 @@ const (
LDAPAuth = "ldap_auth"
ProCrtRestrEveryone = "everyone"
ProCrtRestrAdmOnly = "adminonly"
LDAPScopeBase = "1"
LDAPScopeOnelevel = "2"
LDAPScopeSubtree = "3"
LDAPScopeBase = 1
LDAPScopeOnelevel = 2
LDAPScopeSubtree = 3
RoleProjectAdmin = 1
RoleDeveloper = 2
@ -65,4 +65,5 @@ const (
AdmiralEndpoint = "admiral_url"
WithNotary = "with_notary"
WithClair = "with_clair"
ScanAllPolicy = "scan_all_policy"
)

25
src/common/models/scan_job.go
View File

@ -96,3 +96,28 @@ type VulnerabilityItem struct {
Description string `json:"description"`
Fixed string `json:"fixedVersion,omitempty"`
}
// ScanAllPolicy is represent the json request and object for scan all policy, the parm is het
type ScanAllPolicy struct {
Type string `json:"type"`
Parm map[string]interface{} `json:"parameter, omitempty"`
}
const (
// ScanAllNone "none" for not doing any scan all
ScanAllNone = "none"
// ScanAllDaily for doing scan all daily
ScanAllDaily = "daily"
// ScanAllOnRefresh for doing scan all when the Clair DB is refreshed.
ScanAllOnRefresh = "on_refresh"
// ScanAllDailyTime the key for parm of daily scan all policy.
ScanAllDailyTime = "daily_time"
)
//DefaultScanAllPolicy ...
var DefaultScanAllPolicy = ScanAllPolicy{
Type: ScanAllDaily,
Parm: map[string]interface{}{
ScanAllDailyTime: 0,
},
}

87
src/common/utils/registry/auth/tokenauthorizer.go
View File

@ -15,20 +15,13 @@
package auth
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"sync"
"time"
//"github.com/vmware/harbor/src/common/config"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/common/utils/registry"
registry_error "github.com/vmware/harbor/src/common/utils/error"
token_util "github.com/vmware/harbor/src/ui/service/token"
)
@ -36,13 +29,14 @@ const (
latency int = 10 //second, the network latency when token is received
)
type scope struct {
// Scope ...
type Scope struct {
Type string
Name string
Actions []string
}
func (s *scope) string() string {
func (s *Scope) string() string {
return fmt.Sprintf("%s:%s:%s", s.Type, s.Name, strings.Join(s.Actions, ","))
}
@ -50,7 +44,7 @@ type tokenGenerator func(realm, service string, scopes []string) (token string,
// Implements interface Authorizer
type tokenAuthorizer struct {
scope *scope
scope *Scope
tg tokenGenerator
cache string // cached token
expiresAt *time.Time // The UTC standard time at when the token will expire
@ -64,13 +58,13 @@ func (t *tokenAuthorizer) Scheme() string {
// AuthorizeRequest will add authorization header which contains a token before the request is sent
func (t *tokenAuthorizer) Authorize(req *http.Request, params map[string]string) error {
var scopes []*scope
var scopes []*Scope
var token string
hasFrom := false
from := req.URL.Query().Get("from")
if len(from) != 0 {
s := &scope{
s := &Scope{
Type: "repository",
Name: from,
Actions: []string{"pull"},
@ -154,7 +148,7 @@ func NewStandardTokenAuthorizer(credential Credential, insecure bool,
}
if len(scopeType) != 0 || len(scopeName) != 0 {
authorizer.scope = &scope{
authorizer.scope = &Scope{
Type: scopeType,
Name: scopeName,
Actions: scopeActions,
@ -166,66 +160,21 @@ func NewStandardTokenAuthorizer(credential Credential, insecure bool,
return authorizer
}
func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes []string) (string, int, *time.Time, error) {
realm = s.tokenURL(realm)
tk, err := getToken(s.client, s.credential, realm,
service, scopes)
u, err := url.Parse(realm)
if len(tk.IssuedAt) == 0 {
return tk.Token, tk.ExpiresIn, nil, nil
}
issuedAt, err := time.Parse(time.RFC3339, tk.IssuedAt)
if err != nil {
return
}
q := u.Query()
q.Add("service", service)
for _, scope := range scopes {
q.Add("scope", scope)
}
u.RawQuery = q.Encode()
r, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return
return "", 0, nil, err
}
if s.credential != nil {
s.credential.AddAuthorization(r)
}
resp, err := s.client.Do(r)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode != http.StatusOK {
err = &registry_error.Error{
StatusCode: resp.StatusCode,
Detail: string(b),
}
return
}
tk := models.Token{}
if err = json.Unmarshal(b, &tk); err != nil {
return
}
token = tk.Token
expiresIn = tk.ExpiresIn
if len(tk.IssuedAt) != 0 {
t, err := time.Parse(time.RFC3339, tk.IssuedAt)
if err != nil {
log.Errorf("error occurred while parsing issued_at: %v", err)
err = nil
} else {
issuedAt = &t
}
}
return
return tk.Token, tk.ExpiresIn, &issuedAt, nil
}
// when the registry client is used inside Harbor, the token request
@ -267,7 +216,7 @@ func newUsernameTokenAuthorizer(notary bool, username, scopeType, scopeName stri
username: username,
}
authorizer.scope = &scope{
authorizer.scope = &Scope{
Type: scopeType,
Name: scopeName,
Actions: scopeActions,

93
src/common/utils/registry/auth/util.go Normal file
View File

@ -0,0 +1,93 @@
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"encoding/json"
"io/ioutil"
"net/http"
"net/url"
"github.com/vmware/harbor/src/common/models"
registry_error "github.com/vmware/harbor/src/common/utils/error"
"github.com/vmware/harbor/src/common/utils/registry"
)
const (
service = "harbor-registry"
)
// GetToken requests a token against the endpoint using credetial provided
func GetToken(endpoint string, insecure bool, credential Credential,
scopes []*Scope) (*models.Token, error) {
client := &http.Client{
Transport: registry.GetHTTPTransport(insecure),
}
scopesStr := []string{}
for _, scope := range scopes {
scopesStr = append(scopesStr, scope.string())
}
return getToken(client, credential, endpoint, service, scopesStr)
}
func getToken(client *http.Client, credential Credential, realm, service string,
scopes []string) (*models.Token, error) {
u, err := url.Parse(realm)
if err != nil {
return nil, err
}
query := u.Query()
query.Add("service", service)
for _, scope := range scopes {
query.Add("scope", scope)
}
u.RawQuery = query.Encode()
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return nil, err
}
if credential != nil {
credential.AddAuthorization(req)
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, &registry_error.Error{
StatusCode: resp.StatusCode,
Detail: string(data),
}
}
token := &models.Token{}
if err = json.Unmarshal(data, token); err != nil {
return nil, err
}
return token, nil
}

42
src/jobservice/utils/utils.go
View File

@ -15,11 +15,8 @@
package utils
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/registry"
@ -64,39 +61,18 @@ func BuildBlobURL(endpoint, repository, digest string) string {
return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repository, digest)
}
//GetTokenForRepo is a temp solution for job handler to get a token for clair.
//TODO: Get rid of it when it can get a token from repository client.
//GetTokenForRepo is used for job handler to get a token for clair.
func GetTokenForRepo(repository string) (string, error) {
u, err := url.Parse(config.InternalTokenServiceEndpoint())
if err != nil {
return "", err
}
q := u.Query()
q.Add("service", "harbor-registry")
q.Add("scope", fmt.Sprintf("repository:%s:pull", repository))
u.RawQuery = q.Encode()
r, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return "", err
}
c := &http.Cookie{Name: models.UISecretCookie, Value: config.JobserviceSecret()}
r.AddCookie(c)
client := &http.Client{}
resp, err := client.Do(r)
credentail := auth.NewCookieCredential(c)
token, err := auth.GetToken(config.InternalTokenServiceEndpoint(), true, credentail, []*auth.Scope{&auth.Scope{
Type: "repository",
Name: repository,
Actions: []string{"pull"},
}})
if err != nil {
return "", err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("Unexpected response from token service, code: %d, %s", resp.StatusCode, string(b))
}
tk := models.Token{}
if err := json.Unmarshal(b, &tk); err != nil {
return "", err
}
return tk.Token, nil
return token.Token, nil
}

216
src/ui/api/config.go
View File

@ -17,10 +17,11 @@ package api
import (
"fmt"
"net/http"
"strconv"
"reflect"
"github.com/vmware/harbor/src/common"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/ui/config"
)
@ -62,6 +63,34 @@ var (
common.CfgExpiration,
common.JobLogDir,
common.AdminInitialPassword,
common.ScanAllPolicy,
}
stringKeys = []string{
common.ExtEndpoint,
common.AUTHMode,
common.DatabaseType,
common.MySQLHost,
common.MySQLUsername,
common.MySQLPassword,
common.MySQLDatabase,
common.SQLiteFile,
common.LDAPURL,
common.LDAPSearchDN,
common.LDAPSearchPwd,
common.LDAPBaseDN,
common.LDAPUID,
common.LDAPFilter,
common.TokenServiceURL,
common.RegistryURL,
common.EmailHost,
common.EmailUsername,
common.EmailPassword,
common.EmailFrom,
common.EmailIdentity,
common.ProjectCreationRestriction,
common.JobLogDir,
common.AdminInitialPassword,
}
numKeys = []string{
@ -131,10 +160,10 @@ func (c *ConfigAPI) Get() {
// Put updates configurations
func (c *ConfigAPI) Put() {
m := map[string]string{}
m := map[string]interface{}{}
c.DecodeJSONReq(&m)
cfg := map[string]string{}
cfg := map[string]interface{}{}
for _, k := range validKeys {
if v, ok := m[k]; ok {
cfg[k] = v
@ -152,35 +181,7 @@ func (c *ConfigAPI) Put() {
c.CustomAbort(http.StatusBadRequest, err.Error())
}
if value, ok := cfg[common.AUTHMode]; ok {
mode, err := config.AuthMode()
if err != nil {
log.Errorf("failed to get auth mode: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if mode != value {
flag, err := authModeCanBeModified()
if err != nil {
log.Errorf("failed to determine whether auth mode can be modified: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if !flag {
c.CustomAbort(http.StatusBadRequest,
fmt.Sprintf("%s can not be modified as new users have been inserted into database",
common.AUTHMode))
}
}
}
result, err := convertForPut(cfg)
if err != nil {
log.Errorf("failed to convert configurations: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if err := config.Upload(result); err != nil {
if err := config.Upload(cfg); err != nil {
log.Errorf("failed to upload configurations: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
@ -199,18 +200,53 @@ func (c *ConfigAPI) Reset() {
}
}
func validateCfg(c map[string]string) (bool, error) {
isSysErr := false
func validateCfg(c map[string]interface{}) (bool, error) {
strMap := map[string]string{}
for _, k := range stringKeys {
if _, ok := c[k]; !ok {
continue
}
if _, ok := c[k].(string); !ok {
return false, fmt.Errorf("Invalid value type, expected string, key: %s, value: %v, type: %v", k, c[k], reflect.TypeOf(c[k]))
}
strMap[k] = c[k].(string)
}
numMap := map[string]int{}
for _, k := range numKeys {
if _, ok := c[k]; !ok {
continue
}
if _, ok := c[k].(float64); !ok {
return false, fmt.Errorf("Invalid value type, expected float64, key: %s, value: %v, type: %v", k, c[k], reflect.TypeOf(c[k]))
}
numMap[k] = int(c[k].(float64))
}
boolMap := map[string]bool{}
for _, k := range boolKeys {
if _, ok := c[k]; !ok {
continue
}
if _, ok := c[k].(bool); !ok {
return false, fmt.Errorf("Invalid value type, expected bool, key: %s, value: %v, type: %v", k, c[k], reflect.TypeOf(c[k]))
}
boolMap[k] = c[k].(bool)
}
mode, err := config.AuthMode()
if err != nil {
isSysErr = true
return isSysErr, err
return true, err
}
if value, ok := c[common.AUTHMode]; ok {
if value, ok := strMap[common.AUTHMode]; ok {
if value != common.DBAuth && value != common.LDAPAuth {
return isSysErr, fmt.Errorf("invalid %s, shoud be %s or %s", common.AUTHMode, common.DBAuth, common.LDAPAuth)
return false, fmt.Errorf("invalid %s, shoud be %s or %s", common.AUTHMode, common.DBAuth, common.LDAPAuth)
}
flag, err := authModeCanBeModified()
if err != nil {
return true, err
}
if mode != value && !flag {
return false, fmt.Errorf("%s can not be modified as new users have been inserted into database", common.AUTHMode)
}
mode = value
}
@ -218,123 +254,70 @@ func validateCfg(c map[string]string) (bool, error) {
if mode == common.LDAPAuth {
ldap, err := config.LDAP()
if err != nil {
isSysErr = true
return isSysErr, err
return true, err
}
if len(ldap.URL) == 0 {
if _, ok := c[common.LDAPURL]; !ok {
return isSysErr, fmt.Errorf("%s is missing", common.LDAPURL)
if _, ok := strMap[common.LDAPURL]; !ok {
return false, fmt.Errorf("%s is missing", common.LDAPURL)
}
}
if len(ldap.BaseDN) == 0 {
if _, ok := c[common.LDAPBaseDN]; !ok {
return isSysErr, fmt.Errorf("%s is missing", common.LDAPBaseDN)
if _, ok := strMap[common.LDAPBaseDN]; !ok {
return false, fmt.Errorf("%s is missing", common.LDAPBaseDN)
}
}
if len(ldap.UID) == 0 {
if _, ok := c[common.LDAPUID]; !ok {
return isSysErr, fmt.Errorf("%s is missing", common.LDAPUID)
if _, ok := strMap[common.LDAPUID]; !ok {
return false, fmt.Errorf("%s is missing", common.LDAPUID)
}
}
if ldap.Scope == 0 {
if _, ok := c[common.LDAPScope]; !ok {
return isSysErr, fmt.Errorf("%s is missing", common.LDAPScope)
if _, ok := numMap[common.LDAPScope]; !ok {
return false, fmt.Errorf("%s is missing", common.LDAPScope)
}
}
}
if ldapURL, ok := c[common.LDAPURL]; ok && len(ldapURL) == 0 {
return isSysErr, fmt.Errorf("%s is empty", common.LDAPURL)
if ldapURL, ok := strMap[common.LDAPURL]; ok && len(ldapURL) == 0 {
return false, fmt.Errorf("%s is empty", common.LDAPURL)
}
if baseDN, ok := c[common.LDAPBaseDN]; ok && len(baseDN) == 0 {
return isSysErr, fmt.Errorf("%s is empty", common.LDAPBaseDN)
if baseDN, ok := strMap[common.LDAPBaseDN]; ok && len(baseDN) == 0 {
return false, fmt.Errorf("%s is empty", common.LDAPBaseDN)
}
if uID, ok := c[common.LDAPUID]; ok && len(uID) == 0 {
return isSysErr, fmt.Errorf("%s is empty", common.LDAPUID)
if uID, ok := strMap[common.LDAPUID]; ok && len(uID) == 0 {
return false, fmt.Errorf("%s is empty", common.LDAPUID)
}
if scope, ok := c[common.LDAPScope]; ok &&
if scope, ok := numMap[common.LDAPScope]; ok &&
scope != common.LDAPScopeBase &&
scope != common.LDAPScopeOnelevel &&
scope != common.LDAPScopeSubtree {
return isSysErr, fmt.Errorf("invalid %s, should be %s, %s or %s",
return false, fmt.Errorf("invalid %s, should be %s, %s or %s",
common.LDAPScope,
common.LDAPScopeBase,
common.LDAPScopeOnelevel,
common.LDAPScopeSubtree)
}
for _, k := range boolKeys {
v, ok := c[k]
if !ok {
continue
for k, n := range numMap {
if n < 0 {
return false, fmt.Errorf("invalid %s: %d", k, n)
}
if v != "0" && v != "1" {
return isSysErr, fmt.Errorf("%s should be %s or %s",
k, "0", "1")
}
}
for _, k := range numKeys {
v, ok := c[k]
if !ok {
continue
}
n, err := strconv.Atoi(v)
if err != nil || n < 0 {
return isSysErr, fmt.Errorf("invalid %s: %s", k, v)
}
if (k == common.EmailPort ||
k == common.MySQLPort) && n > 65535 {
return isSysErr, fmt.Errorf("invalid %s: %s", k, v)
return false, fmt.Errorf("invalid %s: %d", k, n)
}
}
if crt, ok := c[common.ProjectCreationRestriction]; ok &&
if crt, ok := strMap[common.ProjectCreationRestriction]; ok &&
crt != common.ProCrtRestrEveryone &&
crt != common.ProCrtRestrAdmOnly {
return isSysErr, fmt.Errorf("invalid %s, should be %s or %s",
return false, fmt.Errorf("invalid %s, should be %s or %s",
common.ProjectCreationRestriction,
common.ProCrtRestrAdmOnly,
common.ProCrtRestrEveryone)
}
return isSysErr, nil
}
//convert map[string]string to map[string]interface{}
func convertForPut(m map[string]string) (map[string]interface{}, error) {
cfg := map[string]interface{}{}
for k, v := range m {
cfg[k] = v
}
for _, k := range numKeys {
if _, ok := cfg[k]; !ok {
continue
}
v, err := strconv.Atoi(cfg[k].(string))
if err != nil {
return nil, err
}
cfg[k] = v
}
for _, k := range boolKeys {
if _, ok := cfg[k]; !ok {
continue
}
cfg[k] = cfg[k] == "1"
}
return cfg, nil
return false, nil
}
// delete sensitive attrs and add editable field to every attr
@ -347,6 +330,9 @@ func convertForGet(cfg map[string]interface{}) (map[string]*value, error) {
}
}
if _, ok := cfg[common.ScanAllPolicy]; !ok {
cfg[common.ScanAllPolicy] = models.DefaultScanAllPolicy
}
for k, v := range cfg {
result[k] = &value{
Value: v,

4
src/ui/api/config_test.go
View File

@ -60,8 +60,8 @@ func TestPutConfig(t *testing.T) {
assert := assert.New(t)
apiTest := newHarborAPI()
cfg := map[string]string{
common.VerifyRemoteCert: "0",
cfg := map[string]interface{}{
common.VerifyRemoteCert: false,
}
code, err := apiTest.PutConfig(*admin, cfg)

2
src/ui/api/harborapi_test.go
View File

@ -1005,7 +1005,7 @@ func (a testapi) GetConfig(authInfo usrInfo) (int, map[string]*value, error) {
return code, cfg, err
}
func (a testapi) PutConfig(authInfo usrInfo, cfg map[string]string) (int, error) {
func (a testapi) PutConfig(authInfo usrInfo, cfg map[string]interface{}) (int, error) {
_sling := sling.New().Base(a.basePath).Put("/api/configurations").BodyJSON(cfg)
code, _, err := request(_sling, jsonAcceptHeader, authInfo)

27
src/ui/config/config.go
View File

@ -16,6 +16,7 @@ package config
import (
"crypto/tls"
"encoding/json"
"fmt"
"net/http"
"os"
@ -355,7 +356,7 @@ func ClairEndpoint() string {
func AdmiralEndpoint() string {
cfg, err := mg.Get()
if err != nil {
log.Errorf("Failed to get configuration, will return empty string as admiral's endpoint")
log.Errorf("Failed to get configuration, will return empty string as admiral's endpoint, error: %v", err)
return ""
}
@ -365,6 +366,30 @@ func AdmiralEndpoint() string {
return cfg[common.AdmiralEndpoint].(string)
}
// ScanAllPolicy returns the policy which controls the scan all.
func ScanAllPolicy() models.ScanAllPolicy {
var res models.ScanAllPolicy
cfg, err := mg.Get()
if err != nil {
log.Errorf("Failed to get configuration, will return default scan all policy, error: %v", err)
return models.DefaultScanAllPolicy
}
v, ok := cfg[common.ScanAllPolicy]
if !ok {
return models.DefaultScanAllPolicy
}
b, err := json.Marshal(v)
if err != nil {
log.Errorf("Failed to Marshal the value in configuration for Scan All policy, error: %v, returning the default policy", err)
return models.DefaultScanAllPolicy
}
if err := json.Unmarshal(b, &res); err != nil {
log.Errorf("Failed to unmarshal the value in configuration for Scan All policy, error: %v, returning the default policy", err)
return models.DefaultScanAllPolicy
}
return res
}
// WithAdmiral returns a bool to indicate if Harbor's deployed with admiral.
func WithAdmiral() bool {
return len(AdmiralEndpoint()) > 0

5
src/ui/config/config_test.go
View File

@ -155,4 +155,9 @@ func TestConfig(t *testing.T) {
if mode != "db_auth" {
t.Errorf("unexpected mode: %s != %s", mode, "db_auth")
}
if s := ScanAllPolicy(); s.Type != "daily" {
t.Errorf("unexpected scan all policy %v", s)
}
}

66
src/ui/proxy/interceptor_test.go
View File

@ -3,6 +3,7 @@ package proxy
import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/vmware/harbor/src/adminserver/client"
"github.com/vmware/harbor/src/common"
"github.com/vmware/harbor/src/common/models"
notarytest "github.com/vmware/harbor/src/common/utils/notary/test"
@ -19,6 +20,7 @@ import (
var endpoint = "10.117.4.142"
var notaryServer *httptest.Server
var adminServer *httptest.Server
var adminserverClient client.Client
var admiralEndpoint = "http://127.0.0.1:8282"
var token = ""
@ -43,6 +45,7 @@ func TestMain(m *testing.M) {
if err := config.Init(); err != nil {
panic(err)
}
adminserverClient = client.NewClient(adminServer.URL, nil)
result := m.Run()
if result != 0 {
os.Exit(result)
@ -95,19 +98,46 @@ func TestEnvPolicyChecker(t *testing.T) {
if err := os.Setenv("PROJECT_CONTENT_TRUST", "1"); err != nil {
t.Fatalf("Failed to set env variable: %v", err)
}
if err2 := os.Setenv("PROJECT_VULNERABLE", "1"); err2 != nil {
t.Fatalf("Failed to set env variable: %v", err2)
}
if err3 := os.Setenv("PROJECT_SEVERITY", "negligible"); err3 != nil {
t.Fatalf("Failed to set env variable: %v", err3)
}
contentTrustFlag := getPolicyChecker().contentTrustEnabled("whatever")
vulFlag := getPolicyChecker().vulnerableEnabled("whatever")
vulFlag, sev := getPolicyChecker().vulnerablePolicy("whatever")
assert.True(contentTrustFlag)
assert.False(vulFlag)
assert.True(vulFlag)
assert.Equal(sev, models.SevNone)
}
func TestPMSPolicyChecker(t *testing.T) {
var defaultConfigAdmiral = map[string]interface{}{
common.ExtEndpoint: "https://" + endpoint,
common.WithNotary: true,
common.CfgExpiration: 5,
common.AdmiralEndpoint: admiralEndpoint,
}
adminServer, err := utilstest.NewAdminserver(defaultConfigAdmiral)
if err != nil {
panic(err)
}
defer adminServer.Close()
if err := os.Setenv("ADMIN_SERVER_URL", adminServer.URL); err != nil {
panic(err)
}
if err := config.Init(); err != nil {
panic(err)
}
pm := pms.NewProjectManager(http.DefaultClient,
admiralEndpoint, nil)
name := "project_for_test_get_true"
name := "project_for_test_get_sev_low"
id, err := pm.Create(&models.Project{
Name: name,
EnableContentTrust: true,
Name: name,
EnableContentTrust: true,
PreventVulnerableImagesFromRunning: false,
PreventVulnerableImagesFromRunningSeverity: "low",
})
require.Nil(t, err)
defer func(id int64) {
@ -118,29 +148,27 @@ func TestPMSPolicyChecker(t *testing.T) {
project, err := pm.Get(id)
assert.Nil(t, err)
assert.Equal(t, id, project.ProjectID)
server, err2 := utilstest.NewAdminserver(nil)
if err2 != nil {
t.Fatalf("failed to create a mock admin server: %v", err2)
}
defer server.Close()
contentTrustFlag := getPolicyChecker().contentTrustEnabled("project_for_test_get_true")
contentTrustFlag := getPolicyChecker().contentTrustEnabled("project_for_test_get_sev_low")
assert.True(t, contentTrustFlag)
projectVulnerableEnabled, projectVulnerableSeverity := getPolicyChecker().vulnerablePolicy("project_for_test_get_sev_low")
assert.False(t, projectVulnerableEnabled)
assert.Equal(t, projectVulnerableSeverity, models.SevLow)
}
func TestMatchNotaryDigest(t *testing.T) {
assert := assert.New(t)
//The data from common/utils/notary/helper_test.go
img1 := imageInfo{"notary-demo/busybox", "1.0", "notary-demo"}
img2 := imageInfo{"notary-demo/busybox", "2.0", "notary-demo"}
res1, err := matchNotaryDigest(img1, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7")
img1 := imageInfo{"notary-demo/busybox", "1.0", "notary-demo", "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"}
img2 := imageInfo{"notary-demo/busybox", "2.0", "notary-demo", "sha256:12345678"}
res1, err := matchNotaryDigest(img1)
assert.Nil(err, "Unexpected error: %v, image: %#v", err, img1)
assert.True(res1)
res2, err := matchNotaryDigest(img1, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a8")
assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img1)
res2, err := matchNotaryDigest(img2)
assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img2)
assert.False(res2)
res3, err := matchNotaryDigest(img2, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7")
assert.Nil(err, "Unexpected error: %v, image: %#v", err, img2)
assert.False(res3)
}
func TestCopyResp(t *testing.T) {

133
src/ui/proxy/interceptors.go
View File

@ -1,9 +1,12 @@
package proxy
import (
// "github.com/vmware/harbor/src/ui/api"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/clair"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/common/utils/notary"
// "github.com/vmware/harbor/src/ui/api"
"github.com/vmware/harbor/src/ui/config"
"github.com/vmware/harbor/src/ui/projectmanager"
@ -25,6 +28,9 @@ const (
tokenUsername = "admin"
)
// Record the docker deamon raw response.
var rec *httptest.ResponseRecorder
// NotaryEndpoint , exported for testing.
var NotaryEndpoint = config.InternalNotaryEndpoint()
@ -50,8 +56,8 @@ func MatchPullManifest(req *http.Request) (bool, string, string) {
type policyChecker interface {
// contentTrustEnabled returns whether a project has enabled content trust.
contentTrustEnabled(name string) bool
// vulnerableEnabled returns whether a project has enabled content trust.
vulnerableEnabled(name string) bool
// vulnerablePolicy returns whether a project has enabled vulnerable, and the project's severity.
vulnerablePolicy(name string) (bool, models.Severity)
}
//For testing
@ -60,9 +66,8 @@ type envPolicyChecker struct{}
func (ec envPolicyChecker) contentTrustEnabled(name string) bool {
return os.Getenv("PROJECT_CONTENT_TRUST") == "1"
}
func (ec envPolicyChecker) vulnerableEnabled(name string) bool {
// TODO: May need get more information in vulnerable policies.
return os.Getenv("PROJECT_VULNERABBLE") == "1"
func (ec envPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity) {
return os.Getenv("PROJECT_VULNERABLE") == "1", clair.ParseClairSev(os.Getenv("PROJECT_SEVERITY"))
}
type pmsPolicyChecker struct {
@ -77,8 +82,13 @@ func (pc pmsPolicyChecker) contentTrustEnabled(name string) bool {
}
return project.EnableContentTrust
}
func (pc pmsPolicyChecker) vulnerableEnabled(name string) bool {
return true
func (pc pmsPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity) {
project, err := pc.pm.Get(name)
if err != nil {
log.Errorf("Unexpected error when getting the project, error: %v", err)
return true, models.SevUnknown
}
return project.PreventVulnerableImagesFromRunning, clair.ParseClairSev(project.PreventVulnerableImagesFromRunningSeverity)
}
// newPMSPolicyChecker returns an instance of an pmsPolicyChecker
@ -100,7 +110,7 @@ type imageInfo struct {
repository string
tag string
projectName string
// digest string
digest string
}
type urlHandler struct {
@ -119,38 +129,22 @@ func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
http.Error(rw, fmt.Sprintf("Bad repository name: %s", repository), http.StatusBadRequest)
return
}
/*
//Need to get digest of the image.
endpoint, err := config.RegistryURL()
if err != nil {
log.Errorf("Error getting Registry URL: %v", err)
http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError)
return
}
rc, err := api.NewRepositoryClient(endpoint, false, username, repository, "repository", repository, "pull")
if err != nil {
log.Errorf("Error creating repository Client: %v", err)
http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError)
return
}
digest, exist, err := rc.ManifestExist(tag)
if err != nil {
log.Errorf("Failed to get digest for tag: %s, error: %v", tag, err)
http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError)
return
}
*/
rec = httptest.NewRecorder()
uh.next.ServeHTTP(rec, req)
if rec.Result().StatusCode != http.StatusOK {
copyResp(rec, rw)
return
}
digest := rec.Header().Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
img := imageInfo{
repository: repository,
tag: tag,
projectName: components[0],
digest: digest,
}
log.Debugf("image info of the request: %#v", img)
ctx := context.WithValue(req.Context(), imageInfoCtxKey, img)
req = req.WithContext(ctx)
}
uh.next.ServeHTTP(rw, req)
}
@ -170,31 +164,70 @@ func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Reque
cth.next.ServeHTTP(rw, req)
return
}
//May need to update status code, let's use recorder
rec := httptest.NewRecorder()
cth.next.ServeHTTP(rec, req)
if rec.Result().StatusCode != http.StatusOK {
copyResp(rec, rw)
return
}
log.Debugf("showing digest")
digest := rec.Header().Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
log.Debugf("digest: %s", digest)
match, err := matchNotaryDigest(img, digest)
match, err := matchNotaryDigest(img)
if err != nil {
http.Error(rw, "Failed in communication with Notary please check the log", http.StatusInternalServerError)
return
}
if match {
log.Debugf("Passing the response to outter responseWriter")
copyResp(rec, rw)
} else {
if !match {
log.Debugf("digest mismatch, failing the response.")
http.Error(rw, "The image is not signed in Notary.", http.StatusPreconditionFailed)
return
}
cth.next.ServeHTTP(rw, req)
}
func matchNotaryDigest(img imageInfo, digest string) (bool, error) {
type vulnerableHandler struct {
next http.Handler
}
func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
imgRaw := req.Context().Value(imageInfoCtxKey)
if imgRaw == nil || !config.WithClair() {
vh.next.ServeHTTP(rw, req)
return
}
img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo)
projectVulnerableEnabled, projectVulnerableSeverity := getPolicyChecker().vulnerablePolicy(img.projectName)
if !projectVulnerableEnabled {
vh.next.ServeHTTP(rw, req)
return
}
overview, err := dao.GetImgScanOverview(img.digest)
if err != nil {
log.Errorf("failed to get ImgScanOverview with repo: %s, tag: %s, digest: %s. Error: %v", img.repository, img.tag, img.digest, err)
http.Error(rw, "Failed to get ImgScanOverview.", http.StatusPreconditionFailed)
return
}
if overview == nil {
log.Debugf("cannot get the image scan overview info, failing the response.")
http.Error(rw, "Cannot get the image scan overview info.", http.StatusPreconditionFailed)
return
}
imageSev := overview.Sev
if imageSev > int(projectVulnerableSeverity) {
log.Debugf("the image severity is higher then project setting, failing the response.")
http.Error(rw, "The image scan result doesn't pass the project setting.", http.StatusPreconditionFailed)
return
}
vh.next.ServeHTTP(rw, req)
}
type funnelHandler struct {
next http.Handler
}
func (fu funnelHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
imgRaw := req.Context().Value(imageInfoCtxKey)
if imgRaw != nil {
log.Debugf("Return the original response as no the interceptor takes action.")
copyResp(rec, rw)
return
}
fu.next.ServeHTTP(rw, req)
}
func matchNotaryDigest(img imageInfo) (bool, error) {
targets, err := notary.GetInternalTargets(NotaryEndpoint, tokenUsername, img.repository)
if err != nil {
return false, err
@ -206,7 +239,7 @@ func matchNotaryDigest(img imageInfo, digest string) (bool, error) {
if err != nil {
return false, err
}
return digest == d, nil
return img.digest == d, nil
}
}
log.Debugf("image: %#v, not found in notary", img)

2
src/ui/proxy/proxy.go
View File

@ -42,7 +42,7 @@ func Init(urls ...string) error {
}
Proxy = httputil.NewSingleHostReverseProxy(targetURL)
//TODO: add vulnerable interceptor.
handlers = handlerChain{head: urlHandler{next: contentTrustHandler{next: Proxy}}}
handlers = handlerChain{head: urlHandler{next: contentTrustHandler{next: vulnerableHandler{next: funnelHandler{next: Proxy}}}}}
return nil
}