diff --git a/Makefile b/Makefile index 440e5f3cc..8a570d9d5 100644 --- a/Makefile +++ b/Makefile @@ -102,7 +102,7 @@ NOTARYVERSION=v0.5.1 MARIADBVERSION=$(VERSIONTAG) CLAIRVERSION=v2.0.1 CLAIRDBVERSION=$(VERSIONTAG) -MIGRATORVERSION=v1.5.0-test +MIGRATORVERSION=v1.5.0 REDISVERSION=$(VERSIONTAG) #clarity parameters diff --git a/docs/migration_guide.md b/docs/migration_guide.md index e9c3abcd4..5abba7f8f 100644 --- a/docs/migration_guide.md +++ b/docs/migration_guide.md @@ -1,10 +1,12 @@ # Harbor upgrade and database migration guide -When upgrading your existing Habor instance to a newer version, you may need to migrate the data in your database. Refer to [change log](../tools/migration/changelog.md) to find out whether there is any change in the database. If there is, you should go through the database migration process. Since the migration may alter the database schema, you should **always** back up your data before any migration. +When upgrading your existing Habor instance to a newer version, you may need to migrate the data in your database and the settings in harbor.cfg. Refer to [change log](../tools/migration/changelog.md) to find out whether there is any change in the database. If there is, you should go through the database migration process. Since the migration may alter the database schema and the settings of harbor.cfg, you should **always** back up your data before any migration. *If your install Harbor for the first time, or the database version is the same as that of the lastest version, you do not need any database migration.* -**NOTE:** +**NOTE:** +- From v1.5.0 on, the migration tool add support for the harbor.cfg migration, which supports upgrade from v1.2.x, v1.3.x and v1.4.x. + - From v1.2 on, you need to use the release version as the tag of the migrator image. 'latest' is no longer used for new release. - You must back up your data before any data migration. @@ -29,24 +31,45 @@ When upgrading your existing Habor instance to a newer version, you may need to 3. Get the lastest Harbor release package from Github: https://github.com/vmware/harbor/releases -4. Before upgrading Harbor, perform database migration first. The migration tool is delivered as a docker image, so you should pull the image from docker hub. Replace [tag] with the release version of Harbor (e.g. 1.2) in the below command: +4. Before upgrading Harbor, perform migration first. The migration tool is delivered as a docker image, so you should pull the image from docker hub. Replace [tag] with the release version of Harbor (e.g. v1.5.0) in the below command: ``` - docker pull vmware/harbor-db-migrator:[tag] + docker pull vmware/harbor-migrator:[tag] ``` -5. Back up database to a directory such as `/path/to/backup`. You need to create the directory if it does not exist. Also, note that the username and password to access the db are provided via environment variable "DB_USR" and "DB_PWD". +5. Back up database/harbor.cfg to a directory such as `/path/to/backup`. You need to create the directory if it does not exist. Also, note that the username and password to access the db are provided via environment variable "DB_USR" and "DB_PWD". **NOTE:** Upgrade from harbor 1.2 or older to harbor 1.3 must use `vmware/migratorharbor-db-migrator:1.2`. Because DB engine replaced by MariaDB in harbor 1.3 ``` - docker run -ti --rm -e DB_USR=root -e DB_PWD=xxxx -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup vmware/harbor-db-migrator:[tag] backup + docker run -it --rm -e DB_USR=root -e DB_PWD={db_pwd} -v ${harbor_db_path}:/var/lib/mysql -v ${harbor_cfg}:/harbor-migration/harbor-cfg/harbor.cfg -v ${backup_path}:/harbor-migration/backup vmware/harbor-migrator:[tag] backup ``` -6. Upgrade database schema and migrate data. + **NOTE:** By default, the migrator handles the backup for DB or CFG. If you want to backup DB or CFG only, refer to the following commands: + + ``` + docker run -it --rm -e DB_USR=root -e DB_PWD={db_pwd} -v ${harbor_db_path}:/var/lib/mysql -v ${backup_path}:/harbor-migration/backup vmware/harbor-migrator:[tag] --db backup + ``` ``` - docker run -ti --rm -e DB_USR=root -e DB_PWD=xxxx -v /data/database:/var/lib/mysql vmware/harbor-db-migrator:[tag] up head + docker run -it --rm -v ${harbor_cfg}:/harbor-migration/harbor-cfg/harbor.cfg -v ${backup_path}:/harbor-migration/backup vmware/harbor-migrator:[tag] --cfg backup + ``` + +6. Upgrade database schema, harbor.cfg and migrate data. + + ``` + docker run -it --rm -e DB_USR=root -e DB_PWD={db_pwd} -v ${harbor_db_path}:/var/lib/mysql -v ${harbor_cfg}:/harbor-migration/harbor-cfg/harbor.cfg vmware/harbor-migrator:[tag] up + ``` + + **NOTE:** By default, the migrator handles the upgrade for DB and CFG. If you want to upgrade DB or CFG only, + refer to the following commands: + + ``` + docker run -it --rm -e DB_USR=root -e DB_PWD={db_pwd} -v ${harbor_db_path}:/var/lib/mysql vmware/harbor-migrator:[tag] --db up + ``` + + ``` + docker run -it --rm -v ${harbor_cfg}:/harbor-migration/harbor-cfg/harbor.cfg vmware/harbor-migrator:[tag] --cfg up ``` **NOTE:** Some errors like @@ -59,25 +82,7 @@ When upgrading your existing Habor instance to a newer version, you may need to ``` will be occurred during upgrading from harbor 1.2 to harbor 1.3, just ignore them if harbor can start successfully. -7. Unzip the new Harbor package and change to `./harbor` as the working directory. Configure Harbor by modifying the file `harbor.cfg`, - - - Configure Harbor by modifying the file `harbor.cfg`, -you may need to refer to the configuration files you've backed up during step 2. -Refer to [Installation & Configuration Guide ](../docs/installation_guide.md) for more information. -Since the content and format of `harbor.cfg` may have been changed in the new release, **DO NOT directly copy `harbor.cfg` from previous version of Harbor.** - - **IMPORTANT:** If you are upgrading a Harbor instance with LDAP/AD authentication, -you must make sure **auth_mode** is set to **ldap_auth** in `harbor.cfg` before launching the new version of Harbor. Otherwise, users may not be able to log in after the upgrade. - - - To assist you in migrating the `harbor.cfg` file from v0.5.0 to v1.1.x, a script is provided and described as below. For other versions of Harbor, you need to manually migrate the file `harbor.cfg`. - - ``` - cd harbor - ./upgrade --source-loc source_harbor_cfg_loc --source-version 0.5.0 --target-loc target_harbor_cfg_loc --target-version 1.1.x - ``` - **NOTE:** After running the script, make sure you go through `harbor.cfg` to verify all the settings are correct. You can make changes to `harbor.cfg` as needed. - -8. Under the directory `./harbor`, run the `./install.sh` script to install the new Harbor instance. If you choose to install Harbor with components like Notary and/or Clair, refer to [Installation & Configuration Guide](../docs/installation_guide.md) for more information. +7. Under the directory `./harbor`, run the `./install.sh` script to install the new Harbor instance. If you choose to install Harbor with components like Notary and/or Clair, refer to [Installation & Configuration Guide](../docs/installation_guide.md) for more information. ### Roll back from an upgrade For any reason, if you want to roll back to the previous version of Harbor, follow the below steps: @@ -91,8 +96,20 @@ For any reason, if you want to roll back to the previous version of Harbor, foll 2. Restore database from backup file in `/path/to/backup` . ``` - docker run -ti --rm -e DB_USR=root -e DB_PWD=xxxx -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup vmware/harbor-db-migrator:[tag] restore + docker run -it --rm -e DB_USR=root -e DB_PWD={db_pwd} -v ${harbor_db_path}:/var/lib/mysql -v ${harbor_cfg}:/harbor-migration/harbor-cfg/harbor.cfg -v ${backup_path}:/harbor-migration/backup vmware/harbor-migrator:[tag] restore ``` + + **NOTE:** By default, the migrator handles the restore for DB and CFG. If you want to restore DB or CFG only, + refer to the following commands: + + ``` + docker run -it --rm -e DB_USR=root -e DB_PWD={db_pwd} -v ${harbor_db_path}:/var/lib/mysql -v ${backup_path}:/harbor-migration/backup vmware/harbor-migrator:[tag] --db restore + ``` + + ``` + docker run -it --rm -v ${harbor_cfg}:/harbor-migration/harbor-cfg/harbor.cfg -v ${backup_path}:/harbor-migration/backup vmware/harbor-migrator:[tag] --cfg restore + ``` + **NOTE:** Rollback from harbor 1.3 to harbor 1.2 should delete `/data/database` directory first, then create new database directory `docker-compose up -d && docker-compose stop`. And must use `vmware/harbor-db-migrator:1.2` to restore. Because of DB engine change. 3. Remove current Harbor instance. @@ -120,10 +137,6 @@ For any reason, if you want to roll back to the previous version of Harbor, foll ``` ### Migration tool reference -- Use `help` command to show instructions of the migration tool: - - ```docker run --rm -e DB_USR=root -e DB_PWD=xxxx vmware/harbor-db-migrator:[tag] help``` - - Use `test` command to test mysql connection: - ```docker run --rm -e DB_USR=root -e DB_PWD=xxxx -v /data/database:/var/lib/mysql vmware/harbor-db-migrator:[tag] test``` + ```docker run -it --rm -e DB_USR=root -e DB_PWD={db_pwd} -v ${harbor_db_path}:/var/lib/mysql -v ${harbor_cfg}:/harbor-migration/harbor-cfg/harbor.cfg vmware/harbor-migrator:[tag] test``` diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 7a52174ea..f2a002a52 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -459,6 +459,10 @@ paths: format: int64 required: true description: Relevant project ID. + - name: entityname + in: query + type: string + description: The entity name to search. tags: - Products responses: diff --git a/make/common/templates/jobservice/config.yml b/make/common/templates/jobservice/config.yml index 881eca96e..65d3b9541 100644 --- a/make/common/templates/jobservice/config.yml +++ b/make/common/templates/jobservice/config.yml @@ -13,13 +13,13 @@ port: 8080 #Worker pool worker_pool: #Worker concurrency - workers: 50 + workers: $max_job_workers backend: "redis" #Additional config if use 'redis' backend - #TODO: switch to internal redis endpoint and namespace. redis_pool: - host: "redis" - port: 6379 + #redis://[arbitrary_username:password@]ipaddress:port/database_index + #or ipaddress:port[,weight,password,database_index] + redis_url: $redis_url namespace: "harbor_job_service_namespace" #Logger for job logger: diff --git a/make/harbor.cfg b/make/harbor.cfg index 86e8743f0..3c06dac7c 100644 --- a/make/harbor.cfg +++ b/make/harbor.cfg @@ -11,7 +11,7 @@ hostname = reg.mydomain.com ui_url_protocol = http #Maximum number of job workers in job service -max_job_workers = 3 +max_job_workers = 50 #Determine whether or not to generate certificate for the registry's token. #If the value is on, the prepare script creates new root cert and private key @@ -141,7 +141,8 @@ db_user = root ##### End of Harbor DB configuration####### #The redis server address. Only needed in HA installation. -redis_url = +#address:port[,weight,password,db_index] +redis_url = redis:6379 ##########Clair DB configuration############ diff --git a/make/prepare b/make/prepare index 60dcccc67..247d04d24 100755 --- a/make/prepare +++ b/make/prepare @@ -412,6 +412,11 @@ render(os.path.join(templates_dir, "jobservice", "env"), ui_secret=ui_secret, jobservice_secret=jobservice_secret, adminserver_url=adminserver_url) + +render(os.path.join(templates_dir, "jobservice", "config.yml"), + jobservice_conf, + max_job_workers=max_job_workers, + redis_url=redis_url) render(os.path.join(templates_dir, "log", "logrotate.conf"), log_rotate_config, @@ -419,7 +424,6 @@ render(os.path.join(templates_dir, "log", "logrotate.conf"), log_rotate_size=log_rotate_size) print("Generated configuration file: %s" % jobservice_conf) -shutil.copyfile(os.path.join(templates_dir, "jobservice", "config.yml"), jobservice_conf) print("Generated configuration file: %s" % ui_conf) shutil.copyfile(os.path.join(templates_dir, "ui", "app.conf"), ui_conf) diff --git a/src/common/dao/project/projectmember.go b/src/common/dao/project/projectmember.go index 91facea00..d78aecd17 100644 --- a/src/common/dao/project/projectmember.go +++ b/src/common/dao/project/projectmember.go @@ -118,3 +118,33 @@ func DeleteProjectMemberByID(pmid int) error { } return nil } + +// SearchMemberByName search members of the project by entity_name +func SearchMemberByName(projectID int64, entityName string) ([]*models.Member, error) { + o := dao.GetOrmer() + sql := `(select pm.id, pm.project_id, + u.username as entity_name, + r.name as rolename, + pm.role, pm.entity_id, pm.entity_type + from project_member pm + left join user u on pm.entity_id = u.user_id and pm.entity_type = 'u' + left join role r on pm.role = r.role_id + where u.deleted = 0 and pm.project_id = ? and u.username like ? order by entity_name ) + union + (select pm.id, pm.project_id, + ug.group_name as entity_name, + r.name as rolename, + pm.role, pm.entity_id, pm.entity_type + from project_member pm + left join user_group ug on pm.entity_id = ug.id and pm.entity_type = 'g' + left join role r on pm.role = r.role_id + where pm.project_id = ? and ug.group_name like ? order by entity_name ) ` + queryParam := make([]interface{}, 4) + queryParam = append(queryParam, projectID) + queryParam = append(queryParam, "%"+dao.Escape(entityName)+"%") + queryParam = append(queryParam, projectID) + queryParam = append(queryParam, "%"+dao.Escape(entityName)+"%") + members := []*models.Member{} + _, err := o.Raw(sql, queryParam).QueryRows(&members) + return members, err +} diff --git a/src/common/dao/project/projectmember_test.go b/src/common/dao/project/projectmember_test.go index 16f0e89c5..bf918b9de 100644 --- a/src/common/dao/project/projectmember_test.go +++ b/src/common/dao/project/projectmember_test.go @@ -149,6 +149,25 @@ func TestAddProjectMember(t *testing.T) { if len(memberList) == 0 { t.Errorf("Failed to query project member, %v", queryMember) } + + _, err = AddProjectMember(models.Member{ + ProjectID: -1, + EntityID: 1, + EntityType: common.UserMember, + Role: models.PROJECTADMIN, + }) + if err == nil { + t.Fatal("Should failed with negative projectID") + } + _, err = AddProjectMember(models.Member{ + ProjectID: 1, + EntityID: -1, + EntityType: common.UserMember, + Role: models.PROJECTADMIN, + }) + if err == nil { + t.Fatal("Should failed with negative entityID") + } } func TestUpdateProjectMemberRole(t *testing.T) { currentProject, err := dao.GetProjectByName("member_test_01") @@ -195,6 +214,23 @@ func TestUpdateProjectMemberRole(t *testing.T) { t.Errorf("member doesn't match!") } + memberList2, err := SearchMemberByName(currentProject.ProjectID, "pm_sample") + if err != nil { + t.Errorf("Error occurred when SearchMemberByName: %v", err) + } + if len(memberList2) == 0 { + t.Errorf("Failed to search user pm_sample, project_id:%v, entityname:%v", + currentProject.ProjectID, "pm_sample") + } + + memberList3, err := SearchMemberByName(currentProject.ProjectID, "") + if err != nil { + t.Errorf("Error occurred when SearchMemberByName: %v", err) + } + if len(memberList3) == 0 { + t.Errorf("Failed to search user pm_sample, project_id:%v, entityname is empty", + currentProject.ProjectID) + } } func TestGetProjectMember(t *testing.T) { diff --git a/src/jobservice/config.yml b/src/jobservice/config.yml index af0e8b081..7bed3c61f 100644 --- a/src/jobservice/config.yml +++ b/src/jobservice/config.yml @@ -17,8 +17,9 @@ worker_pool: backend: "redis" #Additional config if use 'redis' backend redis_pool: - host: "10.160.178.186" - port: 6379 + #redis://[arbitrary_username:password@]ipaddress:port/database_index + #or ipaddress:port[,weight,password,database_index] + redis_url: "redis:6379" namespace: "harbor_job_service" #Logger for job diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index abfcc8d3f..3fb5b820d 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -22,8 +22,7 @@ const ( jobServiceHTTPKey = "JOB_SERVICE_HTTPS_KEY" jobServiceWorkerPoolBackend = "JOB_SERVICE_POOL_BACKEND" jobServiceWorkers = "JOB_SERVICE_POOL_WORKERS" - jobServiceRedisHost = "JOB_SERVICE_POOL_REDIS_HOST" - jobServiceRedisPort = "JOB_SERVICE_POOL_REDIS_PORT" + jobServiceRedisURL = "JOB_SERVICE_POOL_REDIS_URL" jobServiceRedisNamespace = "JOB_SERVICE_POOL_REDIS_NAMESPACE" jobServiceLoggerBasePath = "JOB_SERVICE_LOGGER_BASE_PATH" jobServiceLoggerLevel = "JOB_SERVICE_LOGGER_LEVEL" @@ -41,6 +40,9 @@ const ( //secret of UI uiAuthSecret = "UI_SECRET" + + //redis protocol schema + redisSchema = "redis://" ) //DefaultConfig is the default configuration reference @@ -74,14 +76,13 @@ type HTTPSConfig struct { //RedisPoolConfig keeps redis pool info. type RedisPoolConfig struct { - Host string `yaml:"host"` - Port uint `yaml:"port"` + RedisURL string `yaml:"redis_url"` Namespace string `yaml:"namespace"` } //PoolConfig keeps worker pool configurations. type PoolConfig struct { - //0 means unlimited + //Worker concurrency WorkerCount uint `yaml:"workers"` Backend string `yaml:"backend"` RedisPoolCfg *RedisPoolConfig `yaml:"redis_pool,omitempty"` @@ -118,6 +119,22 @@ func (c *Configuration) Load(yamlFilePath string, detectEnv bool) error { c.loadEnvs() } + //translate redis url if needed + if c.PoolConfig != nil && c.PoolConfig.RedisPoolCfg != nil { + redisAddress := c.PoolConfig.RedisPoolCfg.RedisURL + if !utils.IsEmptyStr(redisAddress) { + if _, err := url.Parse(redisAddress); err != nil { + if redisURL, ok := utils.TranslateRedisAddress(redisAddress); ok { + c.PoolConfig.RedisPoolCfg.RedisURL = redisURL + } + } else { + if !strings.HasPrefix(redisAddress, redisSchema) { + c.PoolConfig.RedisPoolCfg.RedisURL = fmt.Sprintf("%s%s", redisSchema, redisAddress) + } + } + } + } + //Validate settings return c.validate() } @@ -222,22 +239,12 @@ func (c *Configuration) loadEnvs() { } if c.PoolConfig != nil && c.PoolConfig.Backend == JobServicePoolBackendRedis { - rh := utils.ReadEnv(jobServiceRedisHost) - if !utils.IsEmptyStr(rh) { + redisURL := utils.ReadEnv(jobServiceRedisURL) + if !utils.IsEmptyStr(redisURL) { if c.PoolConfig.RedisPoolCfg == nil { c.PoolConfig.RedisPoolCfg = &RedisPoolConfig{} } - c.PoolConfig.RedisPoolCfg.Host = rh - } - - rp := utils.ReadEnv(jobServiceRedisPort) - if !utils.IsEmptyStr(rp) { - if rport, err := strconv.Atoi(rp); err == nil { - if c.PoolConfig.RedisPoolCfg == nil { - c.PoolConfig.RedisPoolCfg = &RedisPoolConfig{} - } - c.PoolConfig.RedisPoolCfg.Port = uint(rport) - } + c.PoolConfig.RedisPoolCfg.RedisURL = redisURL } rn := utils.ReadEnv(jobServiceRedisNamespace) @@ -321,12 +328,18 @@ func (c *Configuration) validate() error { if c.PoolConfig.RedisPoolCfg == nil { return fmt.Errorf("redis pool must be configured when backend is set to '%s'", c.PoolConfig.Backend) } - if utils.IsEmptyStr(c.PoolConfig.RedisPoolCfg.Host) { - return errors.New("host of redis pool is empty") + if utils.IsEmptyStr(c.PoolConfig.RedisPoolCfg.RedisURL) { + return errors.New("URL of redis pool is empty") } - if !utils.IsValidPort(c.PoolConfig.RedisPoolCfg.Port) { - return fmt.Errorf("redis port number should be a none zero integer and less or equal 65535, but current is %d", c.PoolConfig.RedisPoolCfg.Port) + + if !strings.HasPrefix(c.PoolConfig.RedisPoolCfg.RedisURL, redisSchema) { + return errors.New("Invalid redis URL") } + + if _, err := url.Parse(c.PoolConfig.RedisPoolCfg.RedisURL); err != nil { + return fmt.Errorf("Invalid redis URL: %s", err.Error()) + } + if utils.IsEmptyStr(c.PoolConfig.RedisPoolCfg.Namespace) { return errors.New("namespace of redis pool is required") } diff --git a/src/jobservice/config/config_test.go b/src/jobservice/config/config_test.go index abc7c5df6..8c58cd6c5 100644 --- a/src/jobservice/config/config_test.go +++ b/src/jobservice/config/config_test.go @@ -48,11 +48,8 @@ func TestConfigLoadingWithEnv(t *testing.T) { if cfg.PoolConfig.WorkerCount != 8 { t.Fatalf("expect workcount 8 but go '%d'\n", cfg.PoolConfig.WorkerCount) } - if cfg.PoolConfig.RedisPoolCfg.Host != "localhost" { - t.Fatalf("expect redis host 'localhost' but got '%s'\n", cfg.PoolConfig.RedisPoolCfg.Host) - } - if cfg.PoolConfig.RedisPoolCfg.Port != 7379 { - t.Fatalf("expect redis port '7379' but got '%d'\n", cfg.PoolConfig.RedisPoolCfg.Port) + if cfg.PoolConfig.RedisPoolCfg.RedisURL != "redis://arbitrary_username:password@8.8.8.8:6379/0" { + t.Fatalf("expect redis URL 'localhost' but got '%s'\n", cfg.PoolConfig.RedisPoolCfg.RedisURL) } if cfg.PoolConfig.RedisPoolCfg.Namespace != "ut_namespace" { t.Fatalf("expect redis namespace 'ut_namespace' but got '%s'\n", cfg.PoolConfig.RedisPoolCfg.Namespace) @@ -98,6 +95,11 @@ func TestDefaultConfig(t *testing.T) { t.Fatalf("expect default log archive period 1 but got '%d'\n", period) } + redisURL := DefaultConfig.PoolConfig.RedisPoolCfg.RedisURL + if redisURL != "redis://redis:6379" { + t.Fatalf("expect redisURL '%s' but got '%s'\n", "redis://redis:6379", redisURL) + } + if err := RemoveLogDir(); err != nil { t.Fatal(err) } @@ -110,8 +112,7 @@ func setENV() { os.Setenv("JOB_SERVICE_HTTPS_KEY", "../server.key") os.Setenv("JOB_SERVICE_POOL_BACKEND", "redis") os.Setenv("JOB_SERVICE_POOL_WORKERS", "8") - os.Setenv("JOB_SERVICE_POOL_REDIS_HOST", "localhost") - os.Setenv("JOB_SERVICE_POOL_REDIS_PORT", "7379") + os.Setenv("JOB_SERVICE_POOL_REDIS_URL", "8.8.8.8:6379,100,password,0") os.Setenv("JOB_SERVICE_POOL_REDIS_NAMESPACE", "ut_namespace") os.Setenv("JOB_SERVICE_LOGGER_BASE_PATH", "/tmp") os.Setenv("JOB_SERVICE_LOGGER_LEVEL", "DEBUG") @@ -125,8 +126,7 @@ func unsetENV() { os.Unsetenv("JOB_SERVICE_HTTPS_KEY") os.Unsetenv("JOB_SERVICE_POOL_BACKEND") os.Unsetenv("JOB_SERVICE_POOL_WORKERS") - os.Unsetenv("JOB_SERVICE_POOL_REDIS_HOST") - os.Unsetenv("JOB_SERVICE_POOL_REDIS_PORT") + os.Unsetenv("JOB_SERVICE_POOL_REDIS_URL") os.Unsetenv("JOB_SERVICE_POOL_REDIS_NAMESPACE") os.Unsetenv("JOB_SERVICE_LOGGER_BASE_PATH") os.Unsetenv("JOB_SERVICE_LOGGER_LEVEL") diff --git a/src/jobservice/config_test.yml b/src/jobservice/config_test.yml index 38776559c..47a19fae3 100644 --- a/src/jobservice/config_test.yml +++ b/src/jobservice/config_test.yml @@ -12,13 +12,14 @@ port: 9443 #Worker pool worker_pool: - #0 means unlimited + #Worker concurrency workers: 10 backend: "redis" #Additional config if use 'redis' backend redis_pool: - host: "10.160.178.186" - port: 6379 + #redis://[arbitrary_username:password@]ipaddress:port/database_index + #or ipaddress:port[,weight,password,database_index] + redis_url: "redis:6379" namespace: "harbor_job_service" #Logger for job diff --git a/src/jobservice/runtime/bootstrap.go b/src/jobservice/runtime/bootstrap.go index c66240fa8..c82bf7810 100644 --- a/src/jobservice/runtime/bootstrap.go +++ b/src/jobservice/runtime/bootstrap.go @@ -4,7 +4,6 @@ package runtime import ( "context" - "fmt" "os" "os/signal" "sync" @@ -151,9 +150,8 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool(ctx *env.Context, cfg *config.Con MaxIdle: 6, Wait: true, Dial: func() (redis.Conn, error) { - return redis.Dial( - "tcp", - fmt.Sprintf("%s:%d", cfg.PoolConfig.RedisPoolCfg.Host, cfg.PoolConfig.RedisPoolCfg.Port), + return redis.DialURL( + cfg.PoolConfig.RedisPoolCfg.RedisURL, redis.DialConnectTimeout(dialConnectionTimeout), redis.DialReadTimeout(dialReadTimeout), redis.DialWriteTimeout(dialWriteTimeout), diff --git a/src/jobservice/utils/utils.go b/src/jobservice/utils/utils.go index db34310b2..1d42ba89a 100644 --- a/src/jobservice/utils/utils.go +++ b/src/jobservice/utils/utils.go @@ -5,8 +5,10 @@ package utils import ( "errors" + "fmt" "net/url" "os" + "strconv" "strings" "github.com/garyburd/redigo/redis" @@ -71,6 +73,40 @@ func IsValidURL(address string) bool { return true } +//TranslateRedisAddress translates the comma format to redis URL +func TranslateRedisAddress(commaFormat string) (string, bool) { + if IsEmptyStr(commaFormat) { + return "", false + } + + sections := strings.Split(commaFormat, ",") + totalSections := len(sections) + if totalSections == 0 { + return "", false + } + + urlParts := []string{} + //section[0] should be host:port + redisURL := fmt.Sprintf("redis://%s", sections[0]) + if _, err := url.Parse(redisURL); err != nil { + return "", false + } + urlParts = append(urlParts, "redis://", sections[0]) + //Ignore weight + //Check password + if totalSections >= 3 && !IsEmptyStr(sections[2]) { + urlParts = []string{urlParts[0], fmt.Sprintf("%s:%s@", "arbitrary_username", sections[2]), urlParts[1]} + } + + if totalSections >= 4 && !IsEmptyStr(sections[3]) { + if _, err := strconv.Atoi(sections[3]); err == nil { + urlParts = append(urlParts, "/", sections[3]) + } + } + + return strings.Join(urlParts, ""), true +} + //JobScore represents the data item with score in the redis db. type JobScore struct { JobBytes []byte diff --git a/src/ui/api/ldap.go b/src/ui/api/ldap.go index bc4c572cd..29cae8823 100644 --- a/src/ui/api/ldap.go +++ b/src/ui/api/ldap.go @@ -118,7 +118,7 @@ func (l *LdapAPI) ImportUser() { } if len(ldapFailedImportUsers) > 0 { - l.HandleNotFound("Import LDAP user have internal error") + l.HandleNotFound("Import LDAP user is not found") l.Data["json"] = ldapFailedImportUsers l.ServeJSON() return diff --git a/src/ui/api/projectmember.go b/src/ui/api/projectmember.go index fd3813699..102ce2811 100644 --- a/src/ui/api/projectmember.go +++ b/src/ui/api/projectmember.go @@ -73,7 +73,7 @@ func (pma *ProjectMemberAPI) Prepare() { pmid, err := pma.GetInt64FromPath(":pmid") if err != nil { - log.Errorf("Failed to get pmid from path, error %v", err) + log.Warningf("Failed to get pmid from path, error %v", err) } if pmid <= 0 && (pma.Ctx.Input.IsPut() || pma.Ctx.Input.IsDelete()) { pma.HandleBadRequest(fmt.Sprintf("The project member id is invalid, pmid:%s", pma.GetStringFromPath(":pmid"))) @@ -89,8 +89,8 @@ func (pma *ProjectMemberAPI) Get() { queryMember.ProjectID = projectID pma.Data["json"] = make([]models.Member, 0) if pma.id == 0 { - //member id not set, return all member of current project - memberList, err := project.GetProjectMember(queryMember) + entityname := pma.GetString("entityname") + memberList, err := project.SearchMemberByName(projectID, entityname) if err != nil { pma.HandleInternalServerError(fmt.Sprintf("Failed to query database for member list, error: %v", err)) return @@ -98,6 +98,7 @@ func (pma *ProjectMemberAPI) Get() { if len(memberList) > 0 { pma.Data["json"] = memberList } + } else { //return a specific member queryMember.ID = pma.id diff --git a/src/ui/api/projectmember_test.go b/src/ui/api/projectmember_test.go index 6f912b413..49b15e2b1 100644 --- a/src/ui/api/projectmember_test.go +++ b/src/ui/api/projectmember_test.go @@ -61,6 +61,15 @@ func TestProjectMemberAPI_Get(t *testing.T) { }, code: http.StatusNotFound, }, + // 404 + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/99999/members/121", + credential: admin, + }, + code: http.StatusNotFound, + }, } runCodeCheckingCases(t, cases...) } @@ -119,6 +128,22 @@ func TestProjectMemberAPI_Post(t *testing.T) { }, code: http.StatusInternalServerError, }, + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/members?entityname=restuser", + credential: admin, + }, + code: http.StatusOK, + }, + &codeCheckingCase{ + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/members", + credential: admin, + }, + code: http.StatusOK, + }, } runCodeCheckingCases(t, cases...) } diff --git a/tests/apitests/api-testing/client/harbor_api_client.go b/tests/apitests/api-testing/client/harbor_api_client.go index bc87576ba..7f392be24 100644 --- a/tests/apitests/api-testing/client/harbor_api_client.go +++ b/tests/apitests/api-testing/client/harbor_api_client.go @@ -4,6 +4,7 @@ import ( "crypto/tls" "crypto/x509" "errors" + "fmt" "io/ioutil" "net/http" "net/url" @@ -135,6 +136,10 @@ func (ac *APIClient) Post(url string, data []byte) error { if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + if err := getErrorMessage(resp); err != nil { + return fmt.Errorf("%s:%s", resp.Status, err.Error()) + } + return errors.New(resp.Status) } @@ -161,6 +166,10 @@ func (ac *APIClient) Delete(url string) error { } if resp.StatusCode != http.StatusOK { + if err := getErrorMessage(resp); err != nil { + return fmt.Errorf("%s:%s", resp.Status, err.Error()) + } + return errors.New(resp.Status) } @@ -177,3 +186,25 @@ func (ac *APIClient) SwitchAccount(username, password string) { ac.config.Username = username ac.config.Password = password } + +//Read error message from response body +func getErrorMessage(resp *http.Response) error { + if resp == nil { + return errors.New("nil response") + } + + if resp.Body == nil || resp.ContentLength == 0 { + //nothing to read + return nil + } + + defer resp.Body.Close() + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + //abandon to read deatiled error message + return nil + } + + return fmt.Errorf("%s", data) +} diff --git a/tests/apitests/api-testing/lib/project.go b/tests/apitests/api-testing/lib/project.go index c00d3e836..e9cdbb9a1 100644 --- a/tests/apitests/api-testing/lib/project.go +++ b/tests/apitests/api-testing/lib/project.go @@ -88,11 +88,8 @@ func (pu *ProjectUtil) CreateProject(projectName string, accessLevel bool) error } url := pu.rootURI + "/api/projects" - if err = pu.testingClient.Post(url, body); err != nil { - return err - } - return nil + return pu.testingClient.Post(url, body) } //DeleteProject : Delete project @@ -108,11 +105,7 @@ func (pu *ProjectUtil) DeleteProject(projectName string) error { url := fmt.Sprintf("%s%s%d", pu.rootURI, "/api/projects/", pid) - if err := pu.testingClient.Delete(url); err != nil { - return err - } - - return nil + return pu.testingClient.Delete(url) } //AssignRole : Assign role to user @@ -128,8 +121,10 @@ func (pu *ProjectUtil) AssignRole(projectName, username string) error { } m := models.Member{ - UserName: username, - Roles: []int{2}, + RoleID: 2, + Member: &models.MemberUser{ + Username: username, + }, } body, err := json.Marshal(&m) @@ -138,20 +133,17 @@ func (pu *ProjectUtil) AssignRole(projectName, username string) error { } url := fmt.Sprintf("%s%s%d%s", pu.rootURI, "/api/projects/", pid, "/members") - if err := pu.testingClient.Post(url, body); err != nil { - return err - } - return nil + return pu.testingClient.Post(url, body) } //RevokeRole : RevokeRole role from user -func (pu *ProjectUtil) RevokeRole(projectName string, uid int) error { +func (pu *ProjectUtil) RevokeRole(projectName string, username string) error { if len(strings.TrimSpace(projectName)) == 0 { return errors.New("Project name is required for revoking role") } - if uid == 0 { + if len(strings.TrimSpace(username)) == 0 { return errors.New("User ID is required for revoking role") } @@ -160,10 +152,42 @@ func (pu *ProjectUtil) RevokeRole(projectName string, uid int) error { return fmt.Errorf("Failed to get project ID with name %s", projectName) } - url := fmt.Sprintf("%s%s%d%s%d", pu.rootURI, "/api/projects/", pid, "/members/", uid) - if err := pu.testingClient.Delete(url); err != nil { + m, err := pu.GetProjectMember(pid, username) + if err != nil { return err } - return nil + url := fmt.Sprintf("%s%s%d%s%d", pu.rootURI, "/api/projects/", pid, "/members/", m.MID) + + return pu.testingClient.Delete(url) +} + +//GetProjectMember : Get the project member by name +func (pu *ProjectUtil) GetProjectMember(pid int, member string) (*models.ExistingMember, error) { + if pid == 0 { + return nil, errors.New("invalid project ID") + } + + if len(strings.TrimSpace(member)) == 0 { + return nil, errors.New("empty member name") + } + + url := fmt.Sprintf("%s/api/projects/%d/members", pu.rootURI, pid) + data, err := pu.testingClient.Get(url) + if err != nil { + return nil, err + } + + members := []*models.ExistingMember{} + if err := json.Unmarshal(data, &members); err != nil { + return nil, err + } + + for _, m := range members { + if m.Name == member { + return m, nil + } + } + + return nil, fmt.Errorf("no member found by the name '%s'", member) } diff --git a/tests/apitests/api-testing/lib/report.go b/tests/apitests/api-testing/lib/report.go index abda88a5f..6b39a552c 100644 --- a/tests/apitests/api-testing/lib/report.go +++ b/tests/apitests/api-testing/lib/report.go @@ -16,7 +16,11 @@ func (r *Report) Passed(caseName string) { //Failed case func (r *Report) Failed(caseName string, err error) { - r.failed = append(r.failed, fmt.Sprintf("%s: [%s] %s", caseName, "FAILED", err.Error())) + errMsg := "" + if err != nil { + errMsg = err.Error() + } + r.failed = append(r.failed, fmt.Sprintf("%s: [%s] %s", caseName, "FAILED", errMsg)) } //Print report diff --git a/tests/apitests/api-testing/models/member.go b/tests/apitests/api-testing/models/member.go index 3d7983afe..e39d3731a 100644 --- a/tests/apitests/api-testing/models/member.go +++ b/tests/apitests/api-testing/models/member.go @@ -2,6 +2,18 @@ package models //Member : For /api/projects/:pid/members type Member struct { - UserName string `json:"username"` - Roles []int `json:"roles"` + RoleID int `json:"role_id"` + Member *MemberUser `json:"member_user"` +} + +//MemberUser ... +type MemberUser struct { + Username string `json:"username"` +} + +//ExistingMember : For GET /api/projects/20/members +type ExistingMember struct { + MID int `json:"id"` + Name string `json:"entity_name"` + RoleID int `json:"role_id"` } diff --git a/tests/apitests/api-testing/models/project.go b/tests/apitests/api-testing/models/project.go index 690d9adf9..05a9ab9fe 100644 --- a/tests/apitests/api-testing/models/project.go +++ b/tests/apitests/api-testing/models/project.go @@ -3,7 +3,7 @@ package models //Project : For /api/projects type Project struct { Name string `json:"project_name"` - Metadata *Metadata `json:"metadata, omitempty"` + Metadata *Metadata `json:"metadata,omitempty"` } //Metadata : Metadata for project diff --git a/tests/apitests/api-testing/tests/suites/suite01/suite.go b/tests/apitests/api-testing/tests/suites/suite01/suite.go index d5b2f317f..8f5ce1f0a 100644 --- a/tests/apitests/api-testing/tests/suites/suite01/suite.go +++ b/tests/apitests/api-testing/tests/suites/suite01/suite.go @@ -86,8 +86,7 @@ func (ccs *ConcourseCiSuite01) Run(onEnvironment *envs.Environment) *lib.Report } //s7 - uid := usr.GetUserID(onEnvironment.Account) - if err := pro.RevokeRole(onEnvironment.TestingProject, uid); err != nil { + if err := pro.RevokeRole(onEnvironment.TestingProject, onEnvironment.Account); err != nil { report.Failed("RevokeRole", err) } else { report.Passed("RevokeRole") diff --git a/tests/apitests/api-testing/tests/suites/suite02/suite.go b/tests/apitests/api-testing/tests/suites/suite02/suite.go index 3a2839c3d..454ea9de9 100644 --- a/tests/apitests/api-testing/tests/suites/suite02/suite.go +++ b/tests/apitests/api-testing/tests/suites/suite02/suite.go @@ -76,9 +76,7 @@ func (ccs *ConcourseCiSuite02) Run(onEnvironment *envs.Environment) *lib.Report } //s6 - usr := lib.NewUserUtil(onEnvironment.RootURI(), onEnvironment.HTTPClient) - uid := usr.GetUserID(onEnvironment.Account) - if err := pro.RevokeRole(onEnvironment.TestingProject, uid); err != nil { + if err := pro.RevokeRole(onEnvironment.TestingProject, onEnvironment.Account); err != nil { report.Failed("RevokeRole", err) } else { report.Passed("RevokeRole")