diff --git a/make/common/templates/adminserver/env b/make/common/templates/adminserver/env index cc1040fb0..d1228bebb 100644 --- a/make/common/templates/adminserver/env +++ b/make/common/templates/adminserver/env @@ -1,6 +1,6 @@ PORT=8080 LOG_LEVEL=debug -EXT_ENDPOINT=$ui_url +EXT_ENDPOINT=$public_url AUTH_MODE=$auth_mode SELF_REGISTRATION=$self_registration LDAP_URL=$ldap_url @@ -22,8 +22,8 @@ MYSQL_PORT=$db_port MYSQL_USR=$db_user MYSQL_PWD=$db_password MYSQL_DATABASE=registry -REGISTRY_URL=http://registry:5000 -TOKEN_SERVICE_URL=http://ui/service/token +REGISTRY_URL=$registry_url +TOKEN_SERVICE_URL=$token_service_url EMAIL_HOST=$email_host EMAIL_PORT=$email_port EMAIL_USR=$email_usr @@ -53,7 +53,9 @@ UAA_ENDPOINT=$uaa_endpoint UAA_CLIENTID=$uaa_clientid UAA_CLIENTSECRET=$uaa_clientsecret UAA_VERIFY_CERT=$uaa_verify_cert -UI_URL=http://ui:8080 -JOBSERVICE_URL=http://jobservice:8080 +UI_URL=$ui_url +JOBSERVICE_URL=$jobservice_url +CLAIR_URL=$clair_url +NOTARY_URL=$notary_url REGISTRY_STORAGE_PROVIDER_NAME=$storage_provider_name READ_ONLY=false diff --git a/make/common/templates/jobservice/env b/make/common/templates/jobservice/env index 3a025569c..c39f1f71c 100644 --- a/make/common/templates/jobservice/env +++ b/make/common/templates/jobservice/env @@ -1,4 +1,4 @@ UI_SECRET=$ui_secret JOBSERVICE_SECRET=$jobservice_secret -ADMINSERVER_URL=http://adminserver:8080 +ADMINSERVER_URL=$adminserver_url GODEBUG=netdns=cgo diff --git a/make/common/templates/registry/config.yml b/make/common/templates/registry/config.yml index e30dfb47c..c39c42aa1 100644 --- a/make/common/templates/registry/config.yml +++ b/make/common/templates/registry/config.yml @@ -20,14 +20,14 @@ http: auth: token: issuer: harbor-token-issuer - realm: $ui_url/service/token + realm: $public_url/service/token rootcertbundle: /etc/registry/root.crt service: harbor-registry notifications: endpoints: - name: harbor disabled: false - url: http://ui:8080/service/notifications + url: $ui_url/service/notifications timeout: 3000ms threshold: 5 backoff: 1s diff --git a/make/common/templates/registry/config_ha.yml b/make/common/templates/registry/config_ha.yml index e4570db2c..46776c8ff 100644 --- a/make/common/templates/registry/config_ha.yml +++ b/make/common/templates/registry/config_ha.yml @@ -31,7 +31,7 @@ http: auth: token: issuer: harbor-token-issuer - realm: $ui_url/service/token + realm: $public_url/service/token rootcertbundle: /etc/registry/root.crt service: harbor-registry @@ -39,7 +39,7 @@ notifications: endpoints: - name: harbor disabled: false - url: http://ui:8080/service/notifications + url: $ui_url/service/notifications timeout: 3000ms threshold: 5 backoff: 1s diff --git a/make/common/templates/ui/env b/make/common/templates/ui/env index 3a7000135..f9129643f 100644 --- a/make/common/templates/ui/env +++ b/make/common/templates/ui/env @@ -3,6 +3,6 @@ CONFIG_PATH=/etc/ui/app.conf UI_SECRET=$ui_secret JOBSERVICE_SECRET=$jobservice_secret GODEBUG=netdns=cgo -ADMINSERVER_URL=http://adminserver:8080 +ADMINSERVER_URL=$adminserver_url UAA_CA_ROOT=/etc/ui/certificates/uaa_ca.pem _REDIS_URL=$redis_url diff --git a/make/harbor.cfg b/make/harbor.cfg index 738fccdca..a7c851b07 100644 --- a/make/harbor.cfg +++ b/make/harbor.cfg @@ -171,3 +171,4 @@ registry_storage_provider_name = filesystem #registry_storage_provider_config is a comma separated "key: value" pairs, e.g. "key1: value, key2: value2". #Refer to https://docs.docker.com/registry/configuration/#storage for all available configuration. registry_storage_provider_config = + diff --git a/make/photon/db/registry.sql b/make/photon/db/registry.sql index b59694c27..8a4d1b5c8 100644 --- a/make/photon/db/registry.sql +++ b/make/photon/db/registry.sql @@ -119,7 +119,7 @@ create table user_group id int NOT NULL AUTO_INCREMENT, group_name varchar(255) NOT NULL, group_type int default 0, -group_property varchar(512) NOT NULL, +ldap_group_dn varchar(512) NOT NULL, creation_time timestamp default CURRENT_TIMESTAMP, update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, PRIMARY KEY (id) @@ -193,6 +193,8 @@ create table replication_job ( repository varchar(256) NOT NULL, operation varchar(64) NOT NULL, tags varchar(16384), + #New job service only records uuid, for compatibility in this table both IDs are stored. + job_uuid varchar(64), creation_time timestamp default CURRENT_TIMESTAMP, update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, PRIMARY KEY (id), @@ -217,6 +219,8 @@ create table img_scan_job ( repository varchar(256) NOT NULL, tag varchar(128) NOT NULL, digest varchar(128), + #New job service only records uuid, for compatibility in this table both IDs are stored. + job_uuid varchar(64), creation_time timestamp default CURRENT_TIMESTAMP, update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, PRIMARY KEY (id) diff --git a/make/photon/db/registry_sqlite.sql b/make/photon/db/registry_sqlite.sql index 40490a4f7..a6e5d8710 100644 --- a/make/photon/db/registry_sqlite.sql +++ b/make/photon/db/registry_sqlite.sql @@ -64,7 +64,7 @@ create table user_group ( id INTEGER PRIMARY KEY, group_name varchar(255) NOT NULL, group_type int default 0, -group_property varchar(512) NOT NULL, +ldap_group_dn varchar(512) NOT NULL, creation_time timestamp default CURRENT_TIMESTAMP, update_time timestamp default CURRENT_TIMESTAMP ); @@ -183,6 +183,7 @@ create table replication_job ( repository varchar(256) NOT NULL, operation varchar(64) NOT NULL, tags varchar(16384), + job_uuid varchar(64), creation_time timestamp default CURRENT_TIMESTAMP, update_time timestamp default CURRENT_TIMESTAMP ); @@ -204,6 +205,7 @@ create table img_scan_job ( repository varchar(256) NOT NULL, tag varchar(128) NOT NULL, digest varchar(64), + job_uuid varchar(64), creation_time timestamp default CURRENT_TIMESTAMP, update_time timestamp default CURRENT_TIMESTAMP ); diff --git a/make/prepare b/make/prepare index eb9d0047b..49e278077 100755 --- a/make/prepare +++ b/make/prepare @@ -195,7 +195,7 @@ reload_config = rcp.get("configuration", "reload_config") if rcp.has_option( "configuration", "reload_config") else "false" hostname = rcp.get("configuration", "hostname") protocol = rcp.get("configuration", "ui_url_protocol") -ui_url = protocol + "://" + hostname +public_url = protocol + "://" + hostname email_identity = rcp.get("configuration", "email_identity") email_host = rcp.get("configuration", "email_server") email_port = rcp.get("configuration", "email_server_port") @@ -270,7 +270,6 @@ storage_provider_name = rcp.get("configuration", "registry_storage_provider_name storage_provider_config = rcp.get("configuration", "registry_storage_provider_config").strip() # yaml requires 1 or more spaces between the key and value storage_provider_config = storage_provider_config.replace(":", ": ", 1) - ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16)) jobservice_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16)) @@ -298,6 +297,14 @@ job_conf_env = os.path.join(config_dir, "jobservice", "env") nginx_conf = os.path.join(config_dir, "nginx", "nginx.conf") cert_dir = os.path.join(config_dir, "nginx", "cert") log_rotate_config = os.path.join(config_dir, "log", "logrotate.conf") +adminserver_url = "http://adminserver:8080" +registry_url = "http://registry:5000" +ui_url = "http://ui:8080" +token_service_url = "http://ui:8080/service/token" + +jobservice_url = "http://jobservice:8080" +clair_url = "http://clair:6060" +notary_url = "http://notary-server:4443" if protocol == "https": target_cert_path = os.path.join(cert_dir, os.path.basename(cert_path)) @@ -317,6 +324,7 @@ else: render(os.path.join(templates_dir, "adminserver", "env"), adminserver_conf_env, reload_config=reload_config, + public_url=public_url, ui_url=ui_url, auth_mode=auth_mode, self_registration=self_registration, @@ -363,14 +371,20 @@ render(os.path.join(templates_dir, "adminserver", "env"), uaa_clientid=uaa_clientid, uaa_clientsecret=uaa_clientsecret, uaa_verify_cert=uaa_verify_cert, - storage_provider_name=storage_provider_name + storage_provider_name=storage_provider_name, + registry_url=registry_url, + token_service_url=token_service_url, + jobservice_url=jobservice_url, + clair_url=clair_url, + notary_url=notary_url ) render(os.path.join(templates_dir, "ui", "env"), ui_conf_env, ui_secret=ui_secret, jobservice_secret=jobservice_secret, - redis_url = redis_url + redis_url = redis_url, + adminserver_url = adminserver_url ) registry_config_file = "config_ha.yml" if args.ha_mode else "config.yml" @@ -385,6 +399,7 @@ storage_provider_info = ('\n' + ' ' * 4).join( render(os.path.join(templates_dir, "registry", registry_config_file), registry_conf, storage_provider_info=storage_provider_info, + public_url=public_url, ui_url=ui_url, redis_url=redis_url) @@ -395,7 +410,8 @@ render(os.path.join(templates_dir, "db", "env"), render(os.path.join(templates_dir, "jobservice", "env"), job_conf_env, ui_secret=ui_secret, - jobservice_secret=jobservice_secret) + jobservice_secret=jobservice_secret, + adminserver_url=adminserver_url) render(os.path.join(templates_dir, "log", "logrotate.conf"), log_rotate_config, @@ -522,7 +538,7 @@ if args.notary_mode: shutil.copy2(os.path.join(notary_temp_dir, "signer-config.json"), notary_config_dir) render(os.path.join(notary_temp_dir, "server-config.json"), os.path.join(notary_config_dir, "server-config.json"), - token_endpoint=ui_url) + token_endpoint=public_url) print("Copying nginx configuration file for notary") shutil.copy2(os.path.join(templates_dir, "nginx", "notary.upstream.conf"), nginx_conf_d) diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index ed5bce093..c80b049ed 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -151,6 +151,9 @@ var ( }, common.UIURL: "UI_URL", common.JobServiceURL: "JOBSERVICE_URL", + common.TokenServiceURL: "TOKEN_SERVICE_URL", + common.ClairURL: "CLAIR_URL", + common.NotaryURL: "NOTARY_URL", common.RegistryStorageProviderName: "REGISTRY_STORAGE_PROVIDER_NAME", common.ReadOnly: &parser{ env: "READ_ONLY", @@ -202,6 +205,12 @@ var ( parse: parseStringToBool, }, common.RegistryStorageProviderName: "REGISTRY_STORAGE_PROVIDER_NAME", + common.UIURL: "UI_URL", + common.JobServiceURL: "JOBSERVICE_URL", + common.RegistryURL: "REGISTRY_URL", + common.TokenServiceURL: "TOKEN_SERVICE_URL", + common.ClairURL: "CLAIR_URL", + common.NotaryURL: "NOTARY_URL", } ) diff --git a/src/common/const.go b/src/common/const.go index 59806e554..1eba447db 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -100,4 +100,10 @@ const ( UserMember = "u" GroupMember = "g" ReadOnly = "read_only" + ClairURL = "clair_url" + NotaryURL = "notary_url" + DefaultAdminserverEndpoint = "http://adminserver:8080" + DefaultJobserviceEndpoint = "http://jobservice:8080" + DefaultUIEndpoint = "http://ui:8080" + DefaultNotaryEndpoint = "http://notary-server:4443" ) diff --git a/src/common/dao/group/usergroup.go b/src/common/dao/group/usergroup.go new file mode 100644 index 000000000..5218a8a97 --- /dev/null +++ b/src/common/dao/group/usergroup.go @@ -0,0 +1,94 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package group + +import ( + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" +) + +// AddUserGroup - Add User Group +func AddUserGroup(userGroup models.UserGroup) (int, error) { + o := dao.GetOrmer() + id, err := o.Insert(&userGroup) + if err != nil { + return 0, err + } + return int(id), err +} + +// QueryUserGroup - Query User Group +func QueryUserGroup(query models.UserGroup) ([]*models.UserGroup, error) { + o := dao.GetOrmer() + sql := `select id, group_name, group_type, ldap_group_dn from user_group where 1=1 ` + sqlParam := make([]interface{}, 1) + groups := []*models.UserGroup{} + if len(query.GroupName) != 0 { + sql += ` and group_name like ? ` + sqlParam = append(sqlParam, `%`+dao.Escape(query.GroupName)+`%`) + } + + if query.GroupType != 0 { + sql += ` and group_type = ? ` + sqlParam = append(sqlParam, query.GroupType) + } + + if len(query.LdapGroupDN) != 0 { + sql += ` and ldap_group_dn = ? ` + sqlParam = append(sqlParam, query.LdapGroupDN) + } + _, err := o.Raw(sql, sqlParam).QueryRows(&groups) + if err != nil { + return nil, err + } + return groups, nil +} + +// GetUserGroup ... +func GetUserGroup(id int) (*models.UserGroup, error) { + userGroup := models.UserGroup{ID: id} + o := dao.GetOrmer() + err := o.Read(&userGroup) + if err != nil { + return nil, err + } + return &userGroup, nil +} + +// DeleteUserGroup ... +func DeleteUserGroup(id int) error { + userGroup := models.UserGroup{ID: id} + o := dao.GetOrmer() + _, err := o.Delete(&userGroup) + if err == nil { + //Delete all related project members + sql := `delete from project_member where entity_id = ? and entity_type='g'` + _, err := o.Raw(sql, id).Exec() + if err != nil { + return err + } + } + return err +} + +// UpdateUserGroupName ... +func UpdateUserGroupName(id int, groupName string) error { + log.Debugf("Updating user_group with id:%v, name:%v", id, groupName) + o := dao.GetOrmer() + sql := "update user_group set group_name = ? where id = ? " + _, err := o.Raw(sql, groupName, id).Exec() + return err +} diff --git a/src/common/dao/group/usergroup_test.go b/src/common/dao/group/usergroup_test.go new file mode 100644 index 000000000..7e65a09d5 --- /dev/null +++ b/src/common/dao/group/usergroup_test.go @@ -0,0 +1,218 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package group + +import ( + "fmt" + "os" + "testing" + + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" +) + +var createdUserGroupID int + +func TestMain(m *testing.M) { + + //databases := []string{"mysql", "sqlite"} + databases := []string{"mysql"} + for _, database := range databases { + log.Infof("run test cases for database: %s", database) + + result := 1 + switch database { + case "mysql": + dao.PrepareTestForMySQL() + case "sqlite": + dao.PrepareTestForSQLite() + default: + log.Fatalf("invalid database: %s", database) + } + + //Extract to test utils + initSqls := []string{ + "insert into user (username, email, password, realname) values ('member_test_01', 'member_test_01@example.com', '123456', 'member_test_01')", + "insert into project (name, owner_id) values ('member_test_01', 1)", + "insert into user_group (group_name, group_type, ldap_group_dn) values ('test_group_01', 1, 'CN=harbor_users,OU=sample,OU=vmware,DC=harbor,DC=com')", + "update project set owner_id = (select user_id from user where username = 'member_test_01') where name = 'member_test_01'", + "insert into project_member (project_id, entity_id, entity_type, role) values ( (select project_id from project where name = 'member_test_01') , (select user_id from user where username = 'member_test_01'), 'u', 1)", + "insert into project_member (project_id, entity_id, entity_type, role) values ( (select project_id from project where name = 'member_test_01') , (select id from user_group where group_name = 'test_group_01'), 'g', 1)", + } + + clearSqls := []string{ + "delete from project where name='member_test_01'", + "delete from user where username='member_test_01' or username='pm_sample'", + "delete from user_group", + "delete from project_member", + } + dao.PrepareTestData(clearSqls, initSqls) + + result = m.Run() + + if result != 0 { + os.Exit(result) + } + } + +} + +func TestAddUserGroup(t *testing.T) { + type args struct { + userGroup models.UserGroup + } + tests := []struct { + name string + args args + want int + wantErr bool + }{ + {"Insert an ldap user group", args{userGroup: models.UserGroup{GroupName: "sample_group", GroupType: 1, LdapGroupDN: "sample_ldap_dn_string"}}, 0, false}, + {"Insert other user group", args{userGroup: models.UserGroup{GroupName: "other_group", GroupType: 3, LdapGroupDN: "other information"}}, 0, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := AddUserGroup(tt.args.userGroup) + if (err != nil) != tt.wantErr { + t.Errorf("AddUserGroup() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got <= 0 { + t.Errorf("Failed to add user group") + } + }) + } +} + +func TestQueryUserGroup(t *testing.T) { + type args struct { + query models.UserGroup + } + tests := []struct { + name string + args args + want int + wantErr bool + }{ + {"Query all user group", args{query: models.UserGroup{GroupName: "test_group_01"}}, 1, false}, + {"Query all ldap group", args{query: models.UserGroup{GroupType: 1}}, 2, false}, + {"Query ldap group with group property", args{query: models.UserGroup{GroupType: 1, LdapGroupDN: "CN=harbor_users,OU=sample,OU=vmware,DC=harbor,DC=com"}}, 1, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := QueryUserGroup(tt.args.query) + if (err != nil) != tt.wantErr { + t.Errorf("QueryUserGroup() error = %v, wantErr %v", err, tt.wantErr) + return + } + if len(got) != tt.want { + t.Errorf("QueryUserGroup() = %v, want %v", len(got), tt.want) + } + }) + } +} + +func TestGetUserGroup(t *testing.T) { + userGroup := models.UserGroup{GroupName: "insert_group", GroupType: 1, LdapGroupDN: "ldap_dn_string"} + result, err := AddUserGroup(userGroup) + if err != nil { + t.Errorf("Error occurred when AddUserGroup: %v", err) + } + createdUserGroupID = result + type args struct { + id int + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {"Get User Group", args{id: result}, "insert_group", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetUserGroup(tt.args.id) + if (err != nil) != tt.wantErr { + t.Errorf("GetUserGroup() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got.GroupName != tt.want { + t.Errorf("GetUserGroup() = %v, want %v", got.GroupName, tt.want) + } + }) + } +} +func TestUpdateUserGroup(t *testing.T) { + if createdUserGroupID == 0 { + fmt.Println("User group doesn't created, skip to test!") + return + } + type args struct { + id int + groupName string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"Update user group", args{id: createdUserGroupID, groupName: "updated_groupname"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fmt.Printf("id=%v", createdUserGroupID) + if err := UpdateUserGroupName(tt.args.id, tt.args.groupName); (err != nil) != tt.wantErr { + t.Errorf("UpdateUserGroup() error = %v, wantErr %v", err, tt.wantErr) + userGroup, err := GetUserGroup(tt.args.id) + if err != nil { + t.Errorf("Error occurred when GetUserGroup: %v", err) + } + if userGroup == nil { + t.Fatalf("Failed to get updated user group") + } + if userGroup.GroupName != tt.args.groupName { + t.Fatalf("Failed to update user group") + } + } + }) + } +} + +func TestDeleteUserGroup(t *testing.T) { + if createdUserGroupID == 0 { + fmt.Println("User group doesn't created, skip to test!") + return + } + + type args struct { + id int + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"Delete existing user group", args{id: createdUserGroupID}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := DeleteUserGroup(tt.args.id); (err != nil) != tt.wantErr { + t.Errorf("DeleteUserGroup() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/src/common/dao/project/projectmember.go b/src/common/dao/project/projectmember.go new file mode 100644 index 000000000..963eb62d6 --- /dev/null +++ b/src/common/dao/project/projectmember.go @@ -0,0 +1,118 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package project + +import ( + "fmt" + + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" +) + +// GetProjectMember gets all members of the project. +func GetProjectMember(queryMember models.Member) ([]*models.Member, error) { + log.Debugf("Query condition %+v", queryMember) + if queryMember.ProjectID == 0 { + return nil, fmt.Errorf("Failed to query project member, query condition %v", queryMember) + } + + o := dao.GetOrmer() + sql := ` select a.* from ((select pm.id as id, pm.project_id as project_id, ug.id as entity_id, ug.group_name as entity_name, ug.creation_time, ug.update_time, r.name as rolename, + r.role_id as role, pm.entity_type as entity_type from user_group ug join project_member pm + on pm.project_id = ? and ug.id = pm.entity_id join role r on pm.role = r.role_id where pm.entity_type = 'g') + union + (select pm.id as id, pm.project_id as project_id, u.user_id as entity_id, u.username as entity_name, u.creation_time, u.update_time, r.name as rolename, + r.role_id as role, pm.entity_type as entity_type from user u join project_member pm + on pm.project_id = ? and u.user_id = pm.entity_id + join role r on pm.role = r.role_id where u.deleted = 0 and pm.entity_type = 'u')) as a where a.project_id = ? ` + + queryParam := make([]interface{}, 1) + // used ProjectID already + queryParam = append(queryParam, queryMember.ProjectID) + queryParam = append(queryParam, queryMember.ProjectID) + queryParam = append(queryParam, queryMember.ProjectID) + + if len(queryMember.Entityname) > 0 { + sql += " and a.entity_name = ? " + queryParam = append(queryParam, queryMember.Entityname) + } + + if len(queryMember.EntityType) == 1 { + sql += " and a.entity_type = ? " + queryParam = append(queryParam, queryMember.EntityType) + } + + if queryMember.EntityID > 0 { + sql += " and a.entity_id = ? " + queryParam = append(queryParam, queryMember.EntityID) + } + if queryMember.ID > 0 { + sql += " and a.id = ? " + queryParam = append(queryParam, queryMember.ID) + } + sql += ` order by a.entity_name ` + members := []*models.Member{} + _, err := o.Raw(sql, queryParam).QueryRows(&members) + + return members, err +} + +// AddProjectMember inserts a record to table project_member +func AddProjectMember(member models.Member) (int, error) { + + log.Debugf("Adding project member %+v", member) + o := dao.GetOrmer() + + if member.EntityID <= 0 { + return 0, fmt.Errorf("Invalid entity_id, member: %+v", member) + } + + if member.ProjectID <= 0 { + return 0, fmt.Errorf("Invalid project_id, member: %+v", member) + } + + sql := "insert into project_member (project_id, entity_id , role, entity_type) values (?, ?, ?, ?)" + r, err := o.Raw(sql, member.ProjectID, member.EntityID, member.Role, member.EntityType).Exec() + if err != nil { + return 0, err + } + pmid, err := r.LastInsertId() + if err != nil { + return 0, err + } + return int(pmid), err +} + +// UpdateProjectMemberRole updates the record in table project_member, only role can be changed +func UpdateProjectMemberRole(pmID int, role int) error { + if role <= 0 || role >= 3 { + return fmt.Errorf("Failed to update project member, role is not in 0,1,2, role:%v", role) + } + o := dao.GetOrmer() + sql := "update project_member set role = ? where id = ? " + _, err := o.Raw(sql, role, pmID).Exec() + return err +} + +// DeleteProjectMemberByID - Delete Project Member by ID +func DeleteProjectMemberByID(pmid int) error { + o := dao.GetOrmer() + sql := "delete from project_member where id = ?" + if _, err := o.Raw(sql, pmid).Exec(); err != nil { + return err + } + return nil +} diff --git a/src/common/dao/project/projectmember_test.go b/src/common/dao/project/projectmember_test.go new file mode 100644 index 000000000..16f0e89c5 --- /dev/null +++ b/src/common/dao/project/projectmember_test.go @@ -0,0 +1,253 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package project + +import ( + "fmt" + "os" + "testing" + + "github.com/vmware/harbor/src/common" + "github.com/vmware/harbor/src/common/dao" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" + _ "github.com/vmware/harbor/src/ui/auth/db" + _ "github.com/vmware/harbor/src/ui/auth/ldap" + cfg "github.com/vmware/harbor/src/ui/config" +) + +func TestMain(m *testing.M) { + + //databases := []string{"mysql", "sqlite"} + databases := []string{"mysql"} + for _, database := range databases { + log.Infof("run test cases for database: %s", database) + + result := 1 + switch database { + case "mysql": + dao.PrepareTestForMySQL() + case "sqlite": + dao.PrepareTestForSQLite() + default: + log.Fatalf("invalid database: %s", database) + } + + //Extract to test utils + initSqls := []string{ + "insert into user (username, email, password, realname) values ('member_test_01', 'member_test_01@example.com', '123456', 'member_test_01')", + "insert into project (name, owner_id) values ('member_test_01', 1)", + "insert into user_group (group_name, group_type, ldap_group_dn) values ('test_group_01', 1, 'CN=harbor_users,OU=sample,OU=vmware,DC=harbor,DC=com')", + "update project set owner_id = (select user_id from user where username = 'member_test_01') where name = 'member_test_01'", + "insert into project_member (project_id, entity_id, entity_type, role) values ( (select project_id from project where name = 'member_test_01') , (select user_id from user where username = 'member_test_01'), 'u', 1)", + "insert into project_member (project_id, entity_id, entity_type, role) values ( (select project_id from project where name = 'member_test_01') , (select id from user_group where group_name = 'test_group_01'), 'g', 1)", + } + + clearSqls := []string{ + "delete from project where name='member_test_01'", + "delete from user where username='member_test_01' or username='pm_sample'", + "delete from user_group", + "delete from project_member", + } + dao.PrepareTestData(clearSqls, initSqls) + cfg.Init() + result = m.Run() + + if result != 0 { + os.Exit(result) + } + } + +} + +func TestDeleteProjectMemberByID(t *testing.T) { + currentProject, err := dao.GetProjectByName("member_test_01") + + if currentProject == nil || err != nil { + fmt.Println("Failed to load project!") + } else { + fmt.Printf("Load project %+v", currentProject) + } + var addMember = models.Member{ + ProjectID: currentProject.ProjectID, + EntityID: 1, + EntityType: common.UserMember, + Role: models.DEVELOPER, + } + + pmid, err := AddProjectMember(addMember) + + if err != nil { + t.Fatalf("Failed to add project member error: %v", err) + } + + type args struct { + pmid int + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"Delete created", args{pmid}, false}, + {"Delete non exist", args{-13}, false}, + {"Delete non exist", args{13}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := DeleteProjectMemberByID(tt.args.pmid); (err != nil) != tt.wantErr { + t.Errorf("DeleteProjectMemberByID() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } + +} +func TestAddProjectMember(t *testing.T) { + + currentProject, err := dao.GetProjectByName("member_test_01") + if err != nil { + t.Errorf("Error occurred when GetProjectByName: %v", err) + } + member := models.Member{ + ProjectID: currentProject.ProjectID, + EntityID: 1, + EntityType: common.UserMember, + Role: models.PROJECTADMIN, + } + + log.Debugf("Current project id %v", currentProject.ProjectID) + pmid, err := AddProjectMember(member) + if err != nil { + t.Errorf("Error occurred in AddProjectMember: %v", err) + } + if pmid == 0 { + t.Errorf("Error add project member, pmid=0") + } + + queryMember := models.Member{ + ProjectID: currentProject.ProjectID, + ID: pmid, + } + + memberList, err := GetProjectMember(queryMember) + if err != nil { + t.Errorf("Failed to query project member, %v, error: %v", queryMember, err) + } + + if len(memberList) == 0 { + t.Errorf("Failed to query project member, %v", queryMember) + } +} +func TestUpdateProjectMemberRole(t *testing.T) { + currentProject, err := dao.GetProjectByName("member_test_01") + user := models.User{ + Username: "pm_sample", + Email: "pm_sample@example.com", + Realname: "pm_sample", + Password: "1234567d", + } + o := dao.GetOrmer() + userID, err := o.Insert(&user) + if err != nil { + t.Errorf("Error occurred when add user: %v", err) + } + member := models.Member{ + ProjectID: currentProject.ProjectID, + EntityID: int(userID), + EntityType: common.UserMember, + Role: models.PROJECTADMIN, + } + + pmid, err := AddProjectMember(member) + if err != nil { + t.Errorf("Error occurred in UpdateProjectMember: %v", err) + } + + UpdateProjectMemberRole(pmid, models.DEVELOPER) + + queryMember := models.Member{ + ProjectID: currentProject.ProjectID, + EntityID: int(userID), + EntityType: common.UserMember, + } + + memberList, err := GetProjectMember(queryMember) + if err != nil { + t.Errorf("Error occurred in GetProjectMember: %v", err) + } + if len(memberList) != 1 { + t.Errorf("Error occurred in Failed, size: %d, condition:%+v", len(memberList), queryMember) + } + memberItem := memberList[0] + if memberItem.Role != models.DEVELOPER || memberItem.Entityname != user.Username { + t.Errorf("member doesn't match!") + } + +} + +func TestGetProjectMember(t *testing.T) { + currentProject, err := dao.GetProjectByName("member_test_01") + if err != nil { + t.Errorf("Error occurred when GetProjectByName: %v", err) + } + var memberList1 = []*models.Member{ + &models.Member{ + ID: 346, + Entityname: "admin", + Rolename: "projectAdmin", + Role: 1, + EntityID: 1, + EntityType: "u"}, + } + var memberList2 = []*models.Member{ + &models.Member{ + ID: 398, + Entityname: "test_group_01", + Rolename: "projectAdmin", + Role: 1, + EntityType: "g"}, + } + type args struct { + queryMember models.Member + } + tests := []struct { + name string + args args + want []*models.Member + wantErr bool + }{ + {"Query default project member", args{models.Member{ProjectID: currentProject.ProjectID, Entityname: "admin"}}, memberList1, false}, + {"Query default project member group", args{models.Member{ProjectID: currentProject.ProjectID, Entityname: "test_group_01"}}, memberList2, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetProjectMember(tt.args.queryMember) + if (err != nil) != tt.wantErr { + t.Errorf("GetProjectMember() error = %v, wantErr %v", err, tt.wantErr) + return + } + if len(got) != 1 { + t.Errorf("Error occurred when query project member") + } + itemGot := got[0] + itemWant := tt.want[0] + + if itemGot.Entityname != itemWant.Entityname || itemGot.Role != itemWant.Role || itemGot.EntityType != itemWant.EntityType { + t.Errorf("test failed, got:%+v, want:%+v", itemGot, itemWant) + } + }) + } + +} diff --git a/src/common/dao/testutils.go b/src/common/dao/testutils.go index 2b0eadd05..06d64e2b8 100644 --- a/src/common/dao/testutils.go +++ b/src/common/dao/testutils.go @@ -15,6 +15,7 @@ package dao import ( + "fmt" "os" "strconv" @@ -103,3 +104,23 @@ func initDatabaseForTest(db *models.Database) { } } } + +// PrepareTestData -- Clean and Create data +func PrepareTestData(clearSqls []string, initSqls []string) { + o := GetOrmer() + + for _, sql := range clearSqls { + fmt.Printf("Exec sql:%v\n", sql) + _, err := o.Raw(sql).Exec() + if err != nil { + fmt.Printf("failed to clear database, sql:%v, error: %v", sql, err) + } + } + + for _, sql := range initSqls { + _, err := o.Raw(sql).Exec() + if err != nil { + fmt.Printf("failed to init database, sql:%v, error: %v", sql, err) + } + } +} diff --git a/src/common/models/base.go b/src/common/models/base.go index a96e3a72e..2442ff406 100644 --- a/src/common/models/base.go +++ b/src/common/models/base.go @@ -34,5 +34,6 @@ func init() { new(ProjectMetadata), new(ConfigEntry), new(Label), - new(ResourceLabel)) + new(ResourceLabel), + new(UserGroup)) } diff --git a/src/common/models/member.go b/src/common/models/member.go index a01a82a81..3dae34ec8 100644 --- a/src/common/models/member.go +++ b/src/common/models/member.go @@ -32,3 +32,11 @@ type UserMember struct { Rolename string `json:"role_name"` Role int `json:"role_id"` } + +// MemberReq - Create Project Member Request +type MemberReq struct { + ProjectID int64 `json:"project_id"` + Role int `json:"role_id,omitempty"` + MemberUser User `json:"member_user,omitempty"` + MemberGroup UserGroup `json:"member_group,omitempty"` +} diff --git a/src/ui/ui_test.go b/src/common/models/usergroup.go similarity index 52% rename from src/ui/ui_test.go rename to src/common/models/usergroup.go index 453775329..d94f09d18 100644 --- a/src/ui/ui_test.go +++ b/src/common/models/usergroup.go @@ -11,4 +11,21 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package main + +package models + +// UserGroupTable is the name of table in DB that holds the user object +const UserGroupTable = "user_group" + +// UserGroup ... +type UserGroup struct { + ID int `orm:"pk;auto;column(id)" json:"id,omitempty"` + GroupName string `orm:"column(group_name)" json:"group_name,omitempty"` + GroupType int `orm:"column(group_type)" json:"group_type,omitempty"` + LdapGroupDN string `orm:"column(ldap_group_dn)" json:"ldap_group_dn,omitempty"` +} + +// TableName ... +func (u *UserGroup) TableName() string { + return UserGroupTable +} diff --git a/src/common/utils/clair/utils.go b/src/common/utils/clair/utils.go index d5ef006bb..be3152fae 100644 --- a/src/common/utils/clair/utils.go +++ b/src/common/utils/clair/utils.go @@ -15,11 +15,9 @@ package clair import ( - "github.com/vmware/harbor/src/common" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" - "fmt" "strings" ) @@ -44,7 +42,7 @@ func ParseClairSev(clairSev string) models.Severity { } // UpdateScanOverview qeuries the vulnerability based on the layerName and update the record in img_scan_overview table based on digest. -func UpdateScanOverview(digest, layerName string, l ...*log.Logger) error { +func UpdateScanOverview(digest, layerName string, clairEndpoint string, l ...*log.Logger) error { var logger *log.Logger if len(l) > 1 { return fmt.Errorf("More than one logger specified") @@ -53,7 +51,7 @@ func UpdateScanOverview(digest, layerName string, l ...*log.Logger) error { } else { logger = log.DefaultLogger() } - client := NewClient(common.DefaultClairEndpoint, logger) + client := NewClient(clairEndpoint, logger) res, err := client.GetResult(layerName) if err != nil { logger.Errorf("Failed to get result from Clair, error: %v", err) diff --git a/src/common/utils/test/adminserver.go b/src/common/utils/test/adminserver.go index 203ae05f2..c3b6ebe13 100644 --- a/src/common/utils/test/adminserver.go +++ b/src/common/utils/test/adminserver.go @@ -76,6 +76,7 @@ var adminServerDefaultConfig = map[string]interface{}{ common.UIURL: "http://myui:8888/", common.JobServiceURL: "http://myjob:8888/", common.ReadOnly: false, + common.NotaryURL: "http://notary-server:4443", } // NewAdminserver returns a mock admin server diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index 4992c2363..c0f615d30 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -46,7 +46,7 @@ func Init() error { adminServerURL := os.Getenv("ADMINSERVER_URL") if len(adminServerURL) == 0 { - adminServerURL = "http://adminserver" + adminServerURL = common.DefaultAdminserverEndpoint } log.Infof("initializing client for adminserver %s ...", adminServerURL) cfg := &client.Config{ @@ -112,7 +112,7 @@ func LocalUIURL() string { cfg, err := mg.Get() if err != nil { log.Warningf("Failed to Get job service UI URL from backend, error: %v, will return default value.") - return "http://ui" + return common.DefaultUIEndpoint } return strings.TrimSuffix(cfg[common.UIURL].(string), "/") @@ -169,5 +169,12 @@ func InternalTokenServiceEndpoint() string { // ClairEndpoint returns the end point of clair instance, by default it's the one deployed within Harbor. func ClairEndpoint() string { - return common.DefaultClairEndpoint -} + cfg, err :=mg.Get() + if err != nil { + return common.DefaultClairEndpoint + } + if cfg[common.ClairURL] == nil { + return common.DefaultClairEndpoint + } + return cfg[common.ClairURL].(string) +} \ No newline at end of file diff --git a/src/jobservice/scan/handlers.go b/src/jobservice/scan/handlers.go index 0850ce6c8..c1d9e3bc8 100644 --- a/src/jobservice/scan/handlers.go +++ b/src/jobservice/scan/handlers.go @@ -134,7 +134,8 @@ func (sh *SummarizeHandler) Enter() (string, error) { logger.Infof("Entered summarize handler") layerName := sh.Context.layers[len(sh.Context.layers)-1].Name logger.Infof("Top layer's name: %s, will use it to get the vulnerability result of image", layerName) - if err := clair.UpdateScanOverview(sh.Context.Digest, layerName); err != nil { + clairURL := config.ClairEndpoint() + if err := clair.UpdateScanOverview(sh.Context.Digest, layerName, clairURL); err != nil { return "", err } return models.JobFinished, nil diff --git a/src/ui/config/config.go b/src/ui/config/config.go index 8bdfcf402..7a0ceb625 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -66,7 +66,7 @@ func Init() error { initKeyProvider() adminServerURL := os.Getenv("ADMINSERVER_URL") if len(adminServerURL) == 0 { - adminServerURL = "http://adminserver" + adminServerURL = common.DefaultAdminserverEndpoint } return InitByURL(adminServerURL) @@ -295,19 +295,18 @@ func InternalJobServiceURL() string { cfg, err := mg.Get() if err != nil { log.Warningf("Failed to Get job service URL from backend, error: %v, will return default value.") - - return "http://jobservice" + return common.DefaultJobserviceEndpoint } if cfg[common.JobServiceURL] == nil { - return "http://jobservice" + return common.DefaultJobserviceEndpoint } return strings.TrimSuffix(cfg[common.JobServiceURL].(string), "/") } // InternalTokenServiceEndpoint returns token service endpoint for internal communication between Harbor containers func InternalTokenServiceEndpoint() string { - uiURL := "http://ui" + uiURL := common.DefaultUIEndpoint cfg, err := mg.Get() if err != nil { log.Warningf("Failed to Get job service UI URL from backend, error: %v, will use default value.") @@ -321,7 +320,15 @@ func InternalTokenServiceEndpoint() string { // InternalNotaryEndpoint returns notary server endpoint for internal communication between Harbor containers // This is currently a conventional value and can be unaccessible when Harbor is not deployed with Notary. func InternalNotaryEndpoint() string { - return "http://notary-server:4443" + cfg, err := mg.Get() + if err != nil { + log.Warningf("Failed to get Notary endpoint from backend, error: %v, will use default value.") + return common.DefaultNotaryEndpoint + } + if cfg[common.NotaryURL] == nil { + return common.DefaultNotaryEndpoint + } + return cfg[common.NotaryURL].(string) } // InitialAdminPassword returns the initial password for administrator @@ -401,7 +408,7 @@ func JobserviceSecret() string { func WithNotary() bool { cfg, err := mg.Get() if err != nil { - log.Errorf("Failed to get configuration, will return WithNotary == false") + log.Warningf("Failed to get configuration, will return WithNotary == false") return false } return cfg[common.WithNotary].(bool) @@ -419,7 +426,12 @@ func WithClair() bool { // ClairEndpoint returns the end point of clair instance, by default it's the one deployed within Harbor. func ClairEndpoint() string { - return common.DefaultClairEndpoint + cfg, err := mg.Get() + if err != nil { + log.Errorf("Failed to get configuration, use default clair endpoint") + return common.DefaultClairEndpoint + } + return cfg[common.ClairURL].(string) } // ClairDB return Clair db info diff --git a/src/ui/proxy/interceptors.go b/src/ui/proxy/interceptors.go index b18d1c155..5c9924d0a 100644 --- a/src/ui/proxy/interceptors.go +++ b/src/ui/proxy/interceptors.go @@ -35,7 +35,7 @@ const ( var rec *httptest.ResponseRecorder // NotaryEndpoint , exported for testing. -var NotaryEndpoint = config.InternalNotaryEndpoint() +var NotaryEndpoint ="" // MatchPullManifest checks if the request looks like a request to pull manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values func MatchPullManifest(req *http.Request) (bool, string, string) { @@ -294,6 +294,9 @@ func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) } func matchNotaryDigest(img imageInfo) (bool, error) { + if NotaryEndpoint == "" { + NotaryEndpoint = config.InternalNotaryEndpoint() + } targets, err := notary.GetInternalTargets(NotaryEndpoint, tokenUsername, img.repository) if err != nil { return false, err diff --git a/src/ui/service/notifications/clair/handler.go b/src/ui/service/notifications/clair/handler.go index 82e8867a2..0dcd852b0 100644 --- a/src/ui/service/notifications/clair/handler.go +++ b/src/ui/service/notifications/clair/handler.go @@ -32,7 +32,7 @@ const ( ) var ( - clairClient = clair.NewClient(config.ClairEndpoint(), nil) + clairClient *clair.Client ) // Handler handles reqeust on /service/notifications/clair/, which listens to clair's notifications. @@ -43,7 +43,10 @@ type Handler struct { // Handle ... func (h *Handler) Handle() { - var ne models.ClairNotificationEnvelope + if clairClient == nil { + clairClient = clair.NewClient(config.ClairEndpoint(), nil) + } + var ne models.ClairNotificationEnvelope if err := json.Unmarshal(h.Ctx.Input.CopyBody(1<<32), &ne); err != nil { log.Errorf("Failed to decode the request: %v", err) return @@ -85,7 +88,7 @@ func (h *Handler) Handle() { return } for _, e := range l { - if err := clair.UpdateScanOverview(e.Digest, e.DetailsKey); err != nil { + if err := clair.UpdateScanOverview(e.Digest, e.DetailsKey,config.ClairEndpoint()); err != nil { log.Errorf("Failed to refresh scan overview for image: %s", e.Digest) } else { log.Debugf("Refreshed scan overview for record with digest: %s", e.Digest) diff --git a/tools/migration/db/changelog.md b/tools/migration/db/changelog.md index 505f47c09..879b549c5 100644 --- a/tools/migration/db/changelog.md +++ b/tools/migration/db/changelog.md @@ -72,3 +72,4 @@ Changelog for harbor database schema - create table `harbor_resource_label` - create table `user_group` - modify table `project_member` use `id` as PK and add column `entity_type` to indicate if the member is user or group. + - add `job_uuid` column to `replication_job` and `img_scan_job`