Merge remote-tracking branch 'upstream/master' into 170427_delete_ownerid

make/common/templates/clair/config.yaml

@@ -0,0 +1,23 @@
+clair:
+  database:
+    type: pgsql
+    options:
+      source: postgresql://postgres:$password@postgres:5432?sslmode=disable
+
+      # Number of elements kept in the cache
+      # Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database.
+      cachesize: 16384
+
+  api:
+    # API server port
+    port: 6060
+    healthport: 6061
+
+    # Deadline before an API request will respond with a 503
+    timeout: 300s
+  updater:
+    interval: 0h
+
+  notifier:
+    attempts: 3
+    renotifyinterval: 2h

make/common/templates/clair/postgres_env

@@ -0,0 +1 @@
+POSTGRES_PASSWORD=$password

make/docker-compose.clair.yml

@@ -0,0 +1,48 @@
+version: '2'
+services:
+  ui:
+    networks:
+      harbor-clair:
+        aliases:
+          - harbor-ui
+  jobservice:
+    networks:
+      - harbor-clair
+  postgres:
+    networks:
+      harbor-clair:
+        aliases:
+          - postgres
+    container_name: clair-db
+    image: postgres:latest
+    restart: always
+    depends_on:
+      - log
+    env_file:
+      ./common/config/clair/postgres_env
+    volumes:
+      - /data/clair-db:/var/lib/postgresql/data
+    logging:
+      driver: "syslog"
+      options:  
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "clair-db"
+  clair:
+    networks:
+      - harbor-clair
+    container_name: clair
+    image: quay.io/coreos/clair:v2.0.0-rc.0
+    restart: always
+    depends_on:
+      - postgres
+    volumes:
+      - ./common/config/clair:/config
+    command: [-config, /config/config.yaml]
+    logging:
+      driver: "syslog"
+      options:  
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "clair"
+networks:
+  harbor-clair:
+    external: false

make/docker-compose.notary.yml

@@ -9,6 +9,7 @@ services:
   notary-server:
     image: vmware/notary-photon:server-0.5.0
     container_name: notary-server
+    restart: always
     networks:
       - notary-mdb
       - notary-sig
@@ -28,6 +29,7 @@ services:
   notary-signer:
     image: vmware/notary-photon:signer-0.5.0
     container_name: notary-signer
+    restart: always
     networks:
       notary-mdb:
       notary-sig:
@@ -49,6 +51,7 @@ services:
   notary-db:
     image: vmware/harbor-notary-db:mariadb-10.1.10
     container_name: notary-db
+    restart: always
     networks:
       notary-mdb:
         aliases:

make/prepare

@@ -95,6 +95,7 @@ def delfile(src):
 parser = argparse.ArgumentParser()
 parser.add_argument('--conf', dest='cfgfile', default=base_dir+'/harbor.cfg',type=str,help="the path of Harbor configuration file")
 parser.add_argument('--with-notary', dest='notary_mode', default=False, action='store_true', help="the Harbor instance is to be deployed with notary")
+parser.add_argument('--with-clair', dest='clair_mode', default=False, action='store_true', help="the Harbor instance is to be deployed with clair")
 args = parser.parse_args()
 
 delfile(config_dir)
@@ -223,7 +224,8 @@ render(os.path.join(templates_dir, "adminserver", "env"),
         jobservice_secret=jobservice_secret,
         token_expiration=token_expiration,
         admiral_url=admiral_url,
-        with_notary=args.notary_mode
+        with_notary=args.notary_mode,
+        scanner=args.clair_mode and "clair" or "none"
 	)
 
 render(os.path.join(templates_dir, "ui", "env"), 
@@ -364,10 +366,18 @@ if args.notary_mode:
             ssl_cert = os.path.join("/etc/nginx/cert", os.path.basename(target_cert_path)),
             ssl_cert_key = os.path.join("/etc/nginx/cert", os.path.basename(target_cert_key_path)))
 
-
     default_alias = get_alias(secretkey_path)
     render(os.path.join(notary_temp_dir, "signer_env"), os.path.join(notary_config_dir, "signer_env"), alias = default_alias)
 
+if args.clair_mode:
+    pg_password = "password"
+    clair_temp_dir = os.path.join(templates_dir, "clair")
+    clair_config_dir = prep_conf_dir(config_dir, "clair")
+    postgres_env = os.path.join(clair_config_dir, "postgres_env") 
+    render(os.path.join(clair_temp_dir, "postgres_env"), postgres_env, password = pg_password)
+    clair_conf = os.path.join(clair_config_dir, "config.yaml")
+    render(os.path.join(clair_temp_dir, "config.yaml"), clair_conf, password = pg_password)
+
 FNULL.close()
 print("The configuration files are ready, please use docker-compose to start the service.")
 

src/common/secret/store.go

@@ -0,0 +1,48 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package secret
+
+const (
+	// AdminserverUser is the name of adminserver user
+	AdminserverUser = "harbor-adminserver"
+	// JobserviceUser is the name of jobservice user
+	JobserviceUser = "harbor-jobservice"
+	// UIUser is the name of ui user
+	UIUser = "harbor-ui"
+)
+
+// Store the secrets and provides methods to validate secrets
+type Store struct {
+	// the key is secret
+	// the value is username
+	secrets map[string]string
+}
+
+// NewStore ...
+func NewStore(secrets map[string]string) *Store {
+	return &Store{
+		secrets: secrets,
+	}
+}
+
+// IsValid returns whether the secret is valid
+func (s *Store) IsValid(secret string) bool {
+	return len(s.GetUsername(secret)) != 0
+}
+
+// GetUsername returns the corresponding username of the secret
+func (s *Store) GetUsername(secret string) string {
+	return s.secrets[secret]
+}

src/common/secret/store_test.go

@@ -0,0 +1,39 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package secret
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsValid(t *testing.T) {
+	store := NewStore(map[string]string{
+		"secret1": "username1",
+	})
+
+	assert.False(t, store.IsValid("invalid_secret"))
+	assert.True(t, store.IsValid("secret1"))
+}
+
+func TestGetUsername(t *testing.T) {
+	store := NewStore(map[string]string{
+		"secret1": "username1",
+	})
+
+	assert.Equal(t, "", store.GetUsername("invalid_secret"))
+	assert.Equal(t, "username1", store.GetUsername("secret1"))
+}

src/common/security/context.go

@@ -18,15 +18,14 @@ package security
 type Context interface {
 	// IsAuthenticated returns whether the context has been authenticated or not
 	IsAuthenticated() bool
+	// GetUsername returns the username of user related to the context
+	GetUsername() string
 	// IsSysAdmin returns whether the user is system admin
 	IsSysAdmin() bool
 	// HasReadPerm returns whether the user has read permission to the project
-	// whose ID is projectID
-	HasReadPerm(projectID int64) bool
+	HasReadPerm(projectIDOrName interface{}) bool
 	// HasWritePerm  returns whether the user has write permission to the project
-	// whose ID is projectID
-	HasWritePerm(projectID int64) bool
+	HasWritePerm(projectIDOrName interface{}) bool
 	// HasAllPerm  returns whether the user has all permissions to the project
-	// whose ID is projectID
-	HasAllPerm(projectID int64) bool
+	HasAllPerm(projectIDOrName interface{}) bool
 }

src/common/security/secret/context.go

@@ -0,0 +1,74 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package secret
+
+import (
+	"github.com/vmware/harbor/src/common/secret"
+)
+
+// SecurityContext implements security.Context interface based on secret store
+type SecurityContext struct {
+	secret string
+	store  *secret.Store
+}
+
+// NewSecurityContext ...
+func NewSecurityContext(secret string, store *secret.Store) *SecurityContext {
+	return &SecurityContext{
+		secret: secret,
+		store:  store,
+	}
+}
+
+// IsAuthenticated returns true if the secret is valid
+func (s *SecurityContext) IsAuthenticated() bool {
+	if s.store == nil {
+		return false
+	}
+	return s.store.IsValid(s.secret)
+}
+
+// GetUsername returns the corresponding username of the secret
+// or null if the secret is invalid
+func (s *SecurityContext) GetUsername() string {
+	if s.store == nil {
+		return ""
+	}
+	return s.store.GetUsername(s.secret)
+}
+
+// IsSysAdmin always returns false
+func (s *SecurityContext) IsSysAdmin() bool {
+	return false
+}
+
+// HasReadPerm returns true if the corresponding user of the secret
+// is jobservice, otherwise returns false
+func (s *SecurityContext) HasReadPerm(projectIDOrName interface{}) bool {
+	if s.store == nil {
+		return false
+	}
+	return s.store.GetUsername(s.secret) == secret.JobserviceUser
+}
+
+// HasWritePerm always returns false
+func (s *SecurityContext) HasWritePerm(projectIDOrName interface{}) bool {
+	return false
+}
+
+// HasAllPerm always returns false
+func (s *SecurityContext) HasAllPerm(projectIDOrName interface{}) bool {
+	return false
+}

src/common/security/secret/context_test.go

@@ -0,0 +1,134 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package secret
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/vmware/harbor/src/common/secret"
+)
+
+func TestIsAuthenticated(t *testing.T) {
+	// secret store is null
+	context := NewSecurityContext("", nil)
+	isAuthenticated := context.IsAuthenticated()
+	assert.False(t, isAuthenticated)
+
+	//invalid secret
+	context = NewSecurityContext("invalid_secret",
+		secret.NewStore(map[string]string{
+			"secret": "username",
+		}))
+	isAuthenticated = context.IsAuthenticated()
+	assert.False(t, isAuthenticated)
+
+	//valid secret
+	context = NewSecurityContext("secret",
+		secret.NewStore(map[string]string{
+			"secret": "username",
+		}))
+	isAuthenticated = context.IsAuthenticated()
+	assert.True(t, isAuthenticated)
+}
+
+func TestGetUsername(t *testing.T) {
+	// secret store is null
+	context := NewSecurityContext("", nil)
+	username := context.GetUsername()
+	assert.Equal(t, "", username)
+
+	//invalid secret
+	context = NewSecurityContext("invalid_secret",
+		secret.NewStore(map[string]string{
+			"secret": "username",
+		}))
+	username = context.GetUsername()
+	assert.Equal(t, "", username)
+
+	//valid secret
+	context = NewSecurityContext("secret",
+		secret.NewStore(map[string]string{
+			"secret": "username",
+		}))
+	username = context.GetUsername()
+	assert.Equal(t, "username", username)
+}
+
+func TestIsSysAdmin(t *testing.T) {
+	context := NewSecurityContext("secret",
+		secret.NewStore(map[string]string{
+			"secret": "username",
+		}))
+	isSysAdmin := context.IsSysAdmin()
+	assert.False(t, isSysAdmin)
+}
+
+func TestHasReadPerm(t *testing.T) {
+	// secret store is null
+	context := NewSecurityContext("", nil)
+	hasReadPerm := context.HasReadPerm("project_name")
+	assert.False(t, hasReadPerm)
+
+	//invalid secret
+	context = NewSecurityContext("invalid_secret",
+		secret.NewStore(map[string]string{
+			"jobservice_secret": secret.JobserviceUser,
+		}))
+	hasReadPerm = context.HasReadPerm("project_name")
+	assert.False(t, hasReadPerm)
+
+	//valid secret, project name
+	context = NewSecurityContext("jobservice_secret",
+		secret.NewStore(map[string]string{
+			"jobservice_secret": secret.JobserviceUser,
+		}))
+	hasReadPerm = context.HasReadPerm("project_name")
+	assert.True(t, hasReadPerm)
+
+	//valid secret, project ID
+	hasReadPerm = context.HasReadPerm(1)
+	assert.True(t, hasReadPerm)
+}
+
+func TestHasWritePerm(t *testing.T) {
+	context := NewSecurityContext("secret",
+		secret.NewStore(map[string]string{
+			"secret": "username",
+		}))
+
+	// project name
+	hasWritePerm := context.HasWritePerm("project_name")
+	assert.False(t, hasWritePerm)
+
+	// project ID
+	hasWritePerm = context.HasWritePerm(1)
+	assert.False(t, hasWritePerm)
+}
+
+func TestHasAllPerm(t *testing.T) {
+	context := NewSecurityContext("secret",
+		secret.NewStore(map[string]string{
+			"secret": "username",
+		}))
+
+	// project name
+	hasAllPerm := context.HasAllPerm("project_name")
+	assert.False(t, hasAllPerm)
+
+	// project ID
+	hasAllPerm = context.HasAllPerm(1)
+	assert.False(t, hasAllPerm)
+}

src/common/utils/email/mail_test.go

@@ -17,8 +17,7 @@ package email
 import (
 	"strings"
 	"testing"
-
-	"github.com/stretchr/testify/assert"
+	//	"github.com/stretchr/testify/assert"
 )
 
 func TestSend(t *testing.T) {
@@ -39,7 +38,9 @@ func TestSend(t *testing.T) {
 	err := Send(addr, identity, username, password,
 		timeout, tls, insecure, from, to,
 		subject, message)
-	assert.Nil(t, err)
+	//bypass the check due to securty policy change on gmail
+	//TODO
+	//assert.Nil(t, err)
 
 	/*not work on travis
 	// non-tls connection
@@ -77,7 +78,9 @@ func TestPing(t *testing.T) {
 	// tls connection
 	err := Ping(addr, identity, username, password,
 		timeout, tls, insecure)
-	assert.Nil(t, err)
+	//bypass the check due to securty policy change on gmail
+	//TODO
+	//assert.Nil(t, err)
 
 	/*not work on travis
 	// non-tls connection

src/ui/filter/security.go

@@ -19,8 +19,8 @@ import (
 	"strings"
 
 	"github.com/astaxie/beego/context"
+	"github.com/vmware/harbor/src/common/security"
 	"github.com/vmware/harbor/src/common/utils/log"
-	"github.com/vmware/harbor/src/ui/security"
 )
 
 const (

src/ui/pms/db/service.go

@@ -12,4 +12,4 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package secret
+package db

src/ui/pms/service.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package project
+
+// PMS is the project mamagement service which abstracts
+// the operations related to projects
+type PMS interface {
+}

src/ui_ng/src/app/project/create-project/create-project.component.html

@@ -19,8 +19,8 @@
             </span>
           </label>
         </div>
-        <div class="form-group" style="padding-left: 135px;">
-          <label class="col-md-3 form-group-label-override">{{'PROJECT.ACCESS_LEVEL' | translate}}</label>
+        <div class="form-group">
+          <label class="col-md-4 form-group-label-override">{{'PROJECT.PUBLIC' | translate}}</label>
           <div class="checkbox-inline">
             <input type="checkbox" id="create_project_public" [(ngModel)]="project.public" name="public">
             <label for="create_project_public"></label>

src/ui_ng/src/app/shared/statictics/statistics-panel.component.html

@@ -12,10 +12,10 @@
             </div>
             <div class="statistic-column-block" style="margin-left: 16px;">
                 <div>
-                    <statistics [data]='originalCopy.my_project_count' [label]='"STATISTICS.INDEX_MY" | translate'></statistics>
+                    <statistics [data]='originalCopy.my_project_count' [label]='"STATISTICS.INDEX_MY_PROJECTS" | translate'></statistics>
                 </div>
                 <div>
-                    <statistics [data]='originalCopy.my_repo_count' [label]='"STATISTICS.INDEX_MY" | translate'></statistics>
+                    <statistics [data]='originalCopy.my_repo_count' [label]='"STATISTICS.INDEX_MY_REPOSITORIES" | translate'></statistics>
                 </div>
             </div>
             <div class="statistic-column-block" style="margin-left: 28px;">

src/ui_ng/src/i18n/lang/en-us-lang.json

@@ -123,8 +123,7 @@
         "PROJECTS": "Projects",
         "NAME": "Project Name",
         "ROLE": "Role",
-        "PUBLIC_OR_PRIVATE": "Public",
-        "ACCESS_LEVEL": "Access Level",
+        "PUBLIC_OR_PRIVATE": "Access Level",
         "REPO_COUNT": "Repositories Count",
         "CREATION_TIME": "Creation Time",
         "PUBLIC": "Public",
@@ -424,6 +423,8 @@
         "PRO_ITEM": "PROJECTS",
         "REPO_ITEM": "REPOSITORIES",
         "INDEX_MY": "MY",
+        "INDEX_MY_PROJECTS": "MY PROJECTS",
+        "INDEX_MY_REPOSITORIES": "MY REPOSITORIES",
         "INDEX_PUB": "PUBLIC",
         "INDEX_TOTAL": "TOTAL",
         "STORAGE": "STORAGE",

src/ui_ng/src/i18n/lang/zh-cn-lang.json

@@ -123,8 +123,7 @@
         "PROJECTS": "项目",
         "NAME": "项目名称",
         "ROLE": "角色",
-        "PUBLIC_OR_PRIVATE": "公开",
-        "ACCESS_LEVEL": "访问级别",
+        "PUBLIC_OR_PRIVATE": "访问级别",
         "REPO_COUNT": "镜像仓库数",
         "CREATION_TIME": "创建时间",
         "PUBLIC": "公开",
@@ -424,6 +423,8 @@
         "PRO_ITEM": "项目",
         "REPO_ITEM": "镜像仓库",
         "INDEX_MY": "私有",
+        "INDEX_MY_PROJECTS": "我的项目",
+        "INDEX_MY_REPOSITORIES": "我的镜像仓库",
         "INDEX_PUB": "公开",
         "INDEX_TOTAL": "总计",
         "STORAGE": "存储",