Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-29 05:35:43 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #7872 from cd1989/config-redis-pool-idletimeout

Browse Source 
Config idle timeout for redis pool to avoid jobservice restarting
This commit is contained in:
Wang Yan 2019-08-27 14:46:01 +08:00 committed by GitHub
commit 39f78ae768
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 39 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
make/photon/prepare/templates/jobservice/config.yml.jinja
View File

@ -20,6 +20,7 @@ worker_pool:
#redis://[arbitrary_username:password@]ipaddress:port/database_index #redis://[arbitrary_username:password@]ipaddress:port/database_index
redis_url: {{redis_url}} redis_url: {{redis_url}}
namespace: "harbor_job_service_namespace" namespace: "harbor_job_service_namespace"
idle_timeout_second: 3600
#Loggers for the running job #Loggers for the running job
job_loggers: job_loggers:
- name: "STD_OUTPUT" # logger backend name, only support "FILE" and "STD_OUTPUT" - name: "STD_OUTPUT" # logger backend name, only support "FILE" and "STD_OUTPUT"

39
src/jobservice/config/config.go
View File

@ -23,21 +23,23 @@ import (
"strconv" "strconv"
"strings" "strings"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/jobservice/common/utils" "github.com/goharbor/harbor/src/jobservice/common/utils"
yaml "gopkg.in/yaml.v2" yaml "gopkg.in/yaml.v2"
) )
const ( const (
jobServiceProtocol = "JOB_SERVICE_PROTOCOL" jobServiceProtocol = "JOB_SERVICE_PROTOCOL"
jobServicePort = "JOB_SERVICE_PORT" jobServicePort = "JOB_SERVICE_PORT"
jobServiceHTTPCert = "JOB_SERVICE_HTTPS_CERT" jobServiceHTTPCert = "JOB_SERVICE_HTTPS_CERT"
jobServiceHTTPKey = "JOB_SERVICE_HTTPS_KEY" jobServiceHTTPKey = "JOB_SERVICE_HTTPS_KEY"
jobServiceWorkerPoolBackend = "JOB_SERVICE_POOL_BACKEND" jobServiceWorkerPoolBackend = "JOB_SERVICE_POOL_BACKEND"
jobServiceWorkers = "JOB_SERVICE_POOL_WORKERS" jobServiceWorkers = "JOB_SERVICE_POOL_WORKERS"
jobServiceRedisURL = "JOB_SERVICE_POOL_REDIS_URL" jobServiceRedisURL = "JOB_SERVICE_POOL_REDIS_URL"
jobServiceRedisNamespace = "JOB_SERVICE_POOL_REDIS_NAMESPACE" jobServiceRedisNamespace = "JOB_SERVICE_POOL_REDIS_NAMESPACE"
jobServiceAuthSecret = "JOBSERVICE_SECRET" jobServiceRedisIdleConnTimeoutSecond = "JOB_SERVICE_POOL_REDIS_CONN_IDLE_TIMEOUT_SECOND"
coreURL = "CORE_URL" jobServiceAuthSecret = "JOBSERVICE_SECRET"
coreURL = "CORE_URL"
// JobServiceProtocolHTTPS points to the 'https' protocol // JobServiceProtocolHTTPS points to the 'https' protocol
JobServiceProtocolHTTPS = "https" JobServiceProtocolHTTPS = "https"
@ -88,6 +90,10 @@ type HTTPSConfig struct {
type RedisPoolConfig struct { type RedisPoolConfig struct {
RedisURL string `yaml:"redis_url"` RedisURL string `yaml:"redis_url"`
Namespace string `yaml:"namespace"` Namespace string `yaml:"namespace"`
// IdleTimeoutSecond closes connections after remaining idle for this duration. If the value
// is zero, then idle connections are not closed. Applications should set
// the timeout to a value less than the server's timeout.
IdleTimeoutSecond int64 `yaml:"idle_timeout_second"`
} }
// PoolConfig keeps worker worker configurations. // PoolConfig keeps worker worker configurations.
@ -247,6 +253,19 @@ func (c *Configuration) loadEnvs() {
} }
c.PoolConfig.RedisPoolCfg.Namespace = rn c.PoolConfig.RedisPoolCfg.Namespace = rn
} }
it := utils.ReadEnv(jobServiceRedisIdleConnTimeoutSecond)
if !utils.IsEmptyStr(it) {
if c.PoolConfig.RedisPoolCfg == nil {
c.PoolConfig.RedisPoolCfg = &RedisPoolConfig{}
}
v, err := strconv.Atoi(it)
if err != nil {
log.Warningf("Invalid idle timeout second: %s, will use 0 instead", it)
} else {
c.PoolConfig.RedisPoolCfg.IdleTimeoutSecond = int64(v)
}
}
} }
} }

17
src/jobservice/runtime/bootstrap.go
View File

@ -23,7 +23,8 @@ import (
"syscall" "syscall"
"time" "time"
"github.com/goharbor/harbor/src/pkg/scheduler" "github.com/gomodule/redigo/redis"
"github.com/pkg/errors"
"github.com/goharbor/harbor/src/jobservice/api" "github.com/goharbor/harbor/src/jobservice/api"
"github.com/goharbor/harbor/src/jobservice/common/utils" "github.com/goharbor/harbor/src/jobservice/common/utils"
@ -44,8 +45,7 @@ import (
"github.com/goharbor/harbor/src/jobservice/worker" "github.com/goharbor/harbor/src/jobservice/worker"
"github.com/goharbor/harbor/src/jobservice/worker/cworker" "github.com/goharbor/harbor/src/jobservice/worker/cworker"
"github.com/goharbor/harbor/src/pkg/retention" "github.com/goharbor/harbor/src/pkg/retention"
"github.com/gomodule/redigo/redis" "github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/pkg/errors"
) )
const ( const (
@ -100,7 +100,7 @@ func (bs *Bootstrap) LoadAndRun(ctx context.Context, cancel context.CancelFunc)
// Add {} to namespace to void slot issue // Add {} to namespace to void slot issue
namespace := fmt.Sprintf("{%s}", cfg.PoolConfig.RedisPoolCfg.Namespace) namespace := fmt.Sprintf("{%s}", cfg.PoolConfig.RedisPoolCfg.Namespace)
// Get redis connection pool // Get redis connection pool
redisPool := bs.getRedisPool(cfg.PoolConfig.RedisPoolCfg.RedisURL) redisPool := bs.getRedisPool(cfg.PoolConfig.RedisPoolCfg)
// Do data migration if necessary // Do data migration if necessary
rdbMigrator := migration.New(redisPool, namespace) rdbMigrator := migration.New(redisPool, namespace)
@ -263,13 +263,14 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool(
} }
// Get a redis connection pool // Get a redis connection pool
func (bs *Bootstrap) getRedisPool(redisURL string) *redis.Pool { func (bs *Bootstrap) getRedisPool(redisPoolConfig *config.RedisPoolConfig) *redis.Pool {
return &redis.Pool{ return &redis.Pool{
MaxIdle: 6, MaxIdle: 6,
Wait: true, Wait: true,
IdleTimeout: time.Duration(redisPoolConfig.IdleTimeoutSecond) * time.Second,
Dial: func() (redis.Conn, error) { Dial: func() (redis.Conn, error) {
return redis.DialURL( return redis.DialURL(
redisURL, redisPoolConfig.RedisURL,
redis.DialConnectTimeout(dialConnectionTimeout), redis.DialConnectTimeout(dialConnectionTimeout),
redis.DialReadTimeout(dialReadTimeout), redis.DialReadTimeout(dialReadTimeout),
redis.DialWriteTimeout(dialWriteTimeout), redis.DialWriteTimeout(dialWriteTimeout),