Merge pull request #7872 from cd1989/config-redis-pool-idletimeout

Config idle timeout for redis pool to avoid jobservice restarting

make/photon/prepare/templates/jobservice/config.yml.jinja

@@ -20,6 +20,7 @@ worker_pool:
     #redis://[arbitrary_username:password@]ipaddress:port/database_index
     redis_url: {{redis_url}}
     namespace: "harbor_job_service_namespace"
+    idle_timeout_second: 3600
 #Loggers for the running job
 job_loggers:
   - name: "STD_OUTPUT" # logger backend name, only support "FILE" and "STD_OUTPUT"

src/jobservice/config/config.go

@@ -23,21 +23,23 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/jobservice/common/utils"
 	yaml "gopkg.in/yaml.v2"
 )
 
 const (
-	jobServiceProtocol          = "JOB_SERVICE_PROTOCOL"
-	jobServicePort              = "JOB_SERVICE_PORT"
-	jobServiceHTTPCert          = "JOB_SERVICE_HTTPS_CERT"
-	jobServiceHTTPKey           = "JOB_SERVICE_HTTPS_KEY"
-	jobServiceWorkerPoolBackend = "JOB_SERVICE_POOL_BACKEND"
-	jobServiceWorkers           = "JOB_SERVICE_POOL_WORKERS"
-	jobServiceRedisURL          = "JOB_SERVICE_POOL_REDIS_URL"
-	jobServiceRedisNamespace    = "JOB_SERVICE_POOL_REDIS_NAMESPACE"
-	jobServiceAuthSecret        = "JOBSERVICE_SECRET"
-	coreURL                     = "CORE_URL"
+	jobServiceProtocol                   = "JOB_SERVICE_PROTOCOL"
+	jobServicePort                       = "JOB_SERVICE_PORT"
+	jobServiceHTTPCert                   = "JOB_SERVICE_HTTPS_CERT"
+	jobServiceHTTPKey                    = "JOB_SERVICE_HTTPS_KEY"
+	jobServiceWorkerPoolBackend          = "JOB_SERVICE_POOL_BACKEND"
+	jobServiceWorkers                    = "JOB_SERVICE_POOL_WORKERS"
+	jobServiceRedisURL                   = "JOB_SERVICE_POOL_REDIS_URL"
+	jobServiceRedisNamespace             = "JOB_SERVICE_POOL_REDIS_NAMESPACE"
+	jobServiceRedisIdleConnTimeoutSecond = "JOB_SERVICE_POOL_REDIS_CONN_IDLE_TIMEOUT_SECOND"
+	jobServiceAuthSecret                 = "JOBSERVICE_SECRET"
+	coreURL                              = "CORE_URL"
 
 	// JobServiceProtocolHTTPS points to the 'https' protocol
 	JobServiceProtocolHTTPS = "https"
@@ -88,6 +90,10 @@ type HTTPSConfig struct {
 type RedisPoolConfig struct {
 	RedisURL  string `yaml:"redis_url"`
 	Namespace string `yaml:"namespace"`
+	// IdleTimeoutSecond closes connections after remaining idle for this duration. If the value
+	// is zero, then idle connections are not closed. Applications should set
+	// the timeout to a value less than the server's timeout.
+	IdleTimeoutSecond int64 `yaml:"idle_timeout_second"`
 }
 
 // PoolConfig keeps worker worker configurations.
@@ -247,6 +253,19 @@ func (c *Configuration) loadEnvs() {
 			}
 			c.PoolConfig.RedisPoolCfg.Namespace = rn
 		}
+
+		it := utils.ReadEnv(jobServiceRedisIdleConnTimeoutSecond)
+		if !utils.IsEmptyStr(it) {
+			if c.PoolConfig.RedisPoolCfg == nil {
+				c.PoolConfig.RedisPoolCfg = &RedisPoolConfig{}
+			}
+			v, err := strconv.Atoi(it)
+			if err != nil {
+				log.Warningf("Invalid idle timeout second: %s, will use 0 instead", it)
+			} else {
+				c.PoolConfig.RedisPoolCfg.IdleTimeoutSecond = int64(v)
+			}
+		}
 	}
 
 }

src/jobservice/runtime/bootstrap.go

@@ -23,7 +23,8 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/goharbor/harbor/src/pkg/scheduler"
+	"github.com/gomodule/redigo/redis"
+	"github.com/pkg/errors"
 
 	"github.com/goharbor/harbor/src/jobservice/api"
 	"github.com/goharbor/harbor/src/jobservice/common/utils"
@@ -44,8 +45,7 @@ import (
 	"github.com/goharbor/harbor/src/jobservice/worker"
 	"github.com/goharbor/harbor/src/jobservice/worker/cworker"
 	"github.com/goharbor/harbor/src/pkg/retention"
-	"github.com/gomodule/redigo/redis"
-	"github.com/pkg/errors"
+	"github.com/goharbor/harbor/src/pkg/scheduler"
 )
 
 const (
@@ -100,7 +100,7 @@ func (bs *Bootstrap) LoadAndRun(ctx context.Context, cancel context.CancelFunc)
 		// Add {} to namespace to void slot issue
 		namespace := fmt.Sprintf("{%s}", cfg.PoolConfig.RedisPoolCfg.Namespace)
 		// Get redis connection pool
-		redisPool := bs.getRedisPool(cfg.PoolConfig.RedisPoolCfg.RedisURL)
+		redisPool := bs.getRedisPool(cfg.PoolConfig.RedisPoolCfg)
 
 		// Do data migration if necessary
 		rdbMigrator := migration.New(redisPool, namespace)
@@ -263,13 +263,14 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool(
 }
 
 // Get a redis connection pool
-func (bs *Bootstrap) getRedisPool(redisURL string) *redis.Pool {
+func (bs *Bootstrap) getRedisPool(redisPoolConfig *config.RedisPoolConfig) *redis.Pool {
 	return &redis.Pool{
-		MaxIdle: 6,
-		Wait:    true,
+		MaxIdle:     6,
+		Wait:        true,
+		IdleTimeout: time.Duration(redisPoolConfig.IdleTimeoutSecond) * time.Second,
 		Dial: func() (redis.Conn, error) {
 			return redis.DialURL(
-				redisURL,
+				redisPoolConfig.RedisURL,
 				redis.DialConnectTimeout(dialConnectionTimeout),
 				redis.DialReadTimeout(dialReadTimeout),
 				redis.DialWriteTimeout(dialWriteTimeout),