fix: support customize cache db for business (#19184)

Support to configure the customized redis db for cache layer and other
misc business for core, by default the behavior is same with
previous(stored in db 0).

Signed-off-by: chlins <chenyuzh@vmware.com>
This commit is contained in:
Chlins Zhang 2023-08-15 17:24:47 +08:00 committed by GitHub
parent 3f1087269e
commit ad3e767d51
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 167 additions and 45 deletions

View File

@ -174,6 +174,17 @@ _version: 2.8.0
# password: notary_server_db_password
# ssl_mode: disable
# Uncomment redis if need to customize redis db
# redis:
# # db_index 0 is for core, it's unchangeable
# # registry_db_index: 1
# # jobservice_db_index: 2
# # trivy_db_index: 5
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
# Uncomment external_redis if using external Redis server
# external_redis:
# # support redis, redis+sentinel
@ -191,6 +202,10 @@ _version: 2.8.0
# jobservice_db_index: 2
# trivy_db_index: 5
# idle_timeout_seconds: 30
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
# uaa:

View File

@ -381,6 +381,49 @@ external_database:
# ssl_mode: disable
{% endif %}
{% if redis is defined %}
redis:
# # db_index 0 is for core, it's unchangeable
{% if redis.registry_db_index is defined %}
registry_db_index: {{ redis.registry_db_index }}
{% else %}
# # registry_db_index: 1
{% endif %}
{% if redis.jobservice_db_index is defined %}
jobservice_db_index: {{ redis.jobservice_db_index }}
{% else %}
# # jobservice_db_index: 2
{% endif %}
{% if redis.trivy_db_index is defined %}
trivy_db_index: {{ redis.trivy_db_index }}
{% else %}
# # trivy_db_index: 5
{% endif %}
{% if redis.harbor_db_index is defined %}
harbor_db_index: {{ redis.harbor_db_index }}
{% else %}
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
{% endif %}
{% if redis.cache_layer_db_index is defined %}
cache_layer_db_index: {{ redis.cache_layer_db_index }}
{% else %}
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% else %}
# Uncomment redis if need to customize redis db
# redis:
# # db_index 0 is for core, it's unchangeable
# # registry_db_index: 1
# # jobservice_db_index: 2
# # trivy_db_index: 5
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% if external_redis is defined %}
external_redis:
# support redis, redis+sentinel
@ -402,6 +445,19 @@ external_redis:
jobservice_db_index: {{ external_redis.jobservice_db_index }}
trivy_db_index: 5
idle_timeout_seconds: 30
{% if external_redis.harbor_db_index is defined %}
harbor_db_index: {{ redis.harbor_db_index }}
{% else %}
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
{% endif %}
{% if external_redis.harbor_db_index is defined %}
cache_layer_db_index: {{ redis.cache_layer_db_index }}
{% else %}
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% else %}
# Umcomments external_redis if using external Redis server
# external_redis:
@ -420,6 +476,10 @@ external_redis:
# jobservice_db_index: 2
# trivy_db_index: 5
# idle_timeout_seconds: 30
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% if uaa is defined %}

View File

@ -1,6 +1,9 @@
CONFIG_PATH=/etc/core/app.conf
UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem
_REDIS_URL_CORE={{redis_url_core}}
{% if redis_url_harbor %}
_REDIS_URL_HARBOR={{redis_url_harbor}}
{% endif %}
SYNC_QUOTA=true
_REDIS_URL_REG={{redis_url_reg}}
@ -86,6 +89,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
{% endif %}
{% if cache.enabled %}
{% if redis_url_cache_layer %}
_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}}
{% endif %}
CACHE_ENABLED=true
CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
{% endif %}

View File

@ -51,6 +51,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
{% if cache.enabled %}
_REDIS_URL_CORE={{redis_url_core}}
{% if redis_url_cache_layer %}
_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}}
{% endif %}
CACHE_ENABLED=true
CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
{% endif %}

View File

@ -306,7 +306,7 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy):
config_dict['external_database'] = False
# update redis configs
config_dict.update(get_redis_configs(configs.get("external_redis", None), with_trivy))
config_dict.update(get_redis_configs(configs.get("redis", None), configs.get("external_redis", None), with_trivy))
# auto generated secret string for core
config_dict['core_secret'] = generate_random_string(16)
@ -400,7 +400,7 @@ def get_redis_url_param(redis=None):
return ""
def get_redis_configs(external_redis=None, with_trivy=True):
def get_redis_configs(internal_redis=None, external_redis=None, with_trivy=True):
"""Returns configs for redis
>>> get_redis_configs()['external_redis']
@ -433,6 +433,8 @@ def get_redis_configs(external_redis=None, with_trivy=True):
>>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False)
True
"""
internal_redis = internal_redis or {}
external_redis = external_redis or {}
configs = dict(external_redis=bool(external_redis))
@ -447,13 +449,22 @@ def get_redis_configs(external_redis=None, with_trivy=True):
'idle_timeout_seconds': 30,
}
# overwriting existing keys by external_redis
redis.update({key: value for (key, value) in external_redis.items() if value})
if len(internal_redis) > 0:
# overwriting existing keys by internal_redis
redis.update({key: value for (key, value) in internal_redis.items() if value})
else:
# overwriting existing keys by external_redis
redis.update({key: value for (key, value) in external_redis.items() if value})
configs['redis_url_core'] = get_redis_url(0, redis)
configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis)
configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis)
if redis.get('harbor_db_index'):
configs['redis_url_harbor'] = get_redis_url(redis['harbor_db_index'], redis)
if redis.get('cache_layer_db_index'):
configs['redis_url_cache_layer'] = get_redis_url(redis['cache_layer_db_index'], redis)
if with_trivy:
configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis)

View File

@ -127,25 +127,35 @@ func main() {
web.BConfig.WebConfig.Session.SessionName = config.SessionCookieName
web.BConfig.MaxMemory = 1 << 35 // (32GB)
web.BConfig.MaxUploadSize = 1 << 35 // (32GB)
redisURL := os.Getenv("_REDIS_URL_CORE")
if len(redisURL) > 0 {
u, err := url.Parse(redisURL)
// the core db used for beego session
redisCoreURL := os.Getenv("_REDIS_URL_CORE")
if len(redisCoreURL) > 0 {
_, err := url.Parse(redisCoreURL)
if err != nil {
panic("bad _REDIS_URL")
panic("bad _REDIS_URL_CORE")
}
// configure the beego session redis
web.BConfig.WebConfig.Session.SessionProvider = session.HarborProviderName
web.BConfig.WebConfig.Session.SessionProviderConfig = redisURL
log.Info("initializing cache ...")
if err := cache.Initialize(u.Scheme, redisURL); err != nil {
log.Fatalf("failed to initialize cache: %v", err)
}
// when config/db init function is called, the cache is not ready,
// enable config cache explicitly when the cache is ready
dbCfg.EnableConfigCache()
web.BConfig.WebConfig.Session.SessionProviderConfig = redisCoreURL
}
log.Info("initializing cache ...")
// the harbor db used for harbor business, use core db if not specified
redisHarborURL := os.Getenv("_REDIS_URL_HARBOR")
if redisHarborURL == "" {
redisHarborURL = redisCoreURL
}
u, err := url.Parse(redisHarborURL)
if err != nil {
panic("bad _REDIS_URL_HARBOR")
}
if err := cache.Initialize(u.Scheme, redisHarborURL); err != nil {
log.Fatalf("failed to initialize cache: %v", err)
}
// when config/db init function is called, the cache is not ready,
// enable config cache explicitly when the cache is ready
dbCfg.EnableConfigCache()
web.AddTemplateExt("htm")
log.Info("initializing configurations...")

View File

@ -19,8 +19,6 @@ import (
"errors"
"flag"
"fmt"
"net/url"
"os"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/jobservice/common/utils"
@ -29,7 +27,6 @@ import (
"github.com/goharbor/harbor/src/jobservice/job/impl"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/jobservice/runtime"
"github.com/goharbor/harbor/src/lib/cache"
cfgLib "github.com/goharbor/harbor/src/lib/config"
tracelib "github.com/goharbor/harbor/src/lib/trace"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
@ -45,21 +42,6 @@ func main() {
panic(fmt.Sprintf("failed to load configuration, error: %v", err))
}
// init cache if cache layer enabled
// gc needs to delete artifact by artifact manager, but the artifact cache store in
// core redis db so here require core redis url and init default cache.
if cfgLib.CacheEnabled() {
cacheURL := os.Getenv("_REDIS_URL_CORE")
u, err := url.Parse(cacheURL)
if err != nil {
panic("bad _REDIS_URL_CORE")
}
if err = cache.Initialize(u.Scheme, cacheURL); err != nil {
panic(fmt.Sprintf("failed to initialize cache: %v", err))
}
}
// Get parameters
configPath := flag.String("c", "", "Specify the yaml config file path")
flag.Parse()

View File

@ -19,6 +19,7 @@ import (
"errors"
"fmt"
"net/url"
"os"
"sync"
"time"
@ -137,3 +138,37 @@ func Initialize(typ, addr string) error {
func Default() Cache {
return cache
}
var (
// cacheLayer is the global cache layer cache instance.
cacheLayer Cache
// cacheLayerOnce is the once condition for initializing instance.
cacheLayerOnce sync.Once
)
// LayerCache is the global cache instance for cache layer.
func LayerCache() Cache {
// parse the redis url for cache layer, use the default cache if not specify
redisCacheURL := os.Getenv("_REDIS_URL_CACHE_LAYER")
if redisCacheURL == "" {
if cache != nil {
return cache
}
// use the core url if cache layer url not found
redisCacheURL = os.Getenv("_REDIS_URL_CORE")
}
u, err := url.Parse(redisCacheURL)
if err != nil {
log.Fatal("failed to parse the redis url for cache layer, bad _REDIS_URL_CACHE_LAYER")
}
cacheLayerOnce.Do(func() {
cacheLayer, err = New(u.Scheme, Address(redisCacheURL), Prefix("cache:"))
if err != nil {
log.Fatalf("failed to initialize cache for cache layer, err: %v", err)
}
})
return cacheLayer
}

View File

@ -24,27 +24,27 @@ import (
)
// innerCache is the default cache client,
// actually it is a wrapper for cache.Default().
// actually it is a wrapper for cache.LayerCache().
var innerCache cache.Cache = &cacheClient{}
// cacheClient is a interceptor for cache.Default, in order to implement specific
// cacheClient is a interceptor for cache.CacheLayer, in order to implement specific
// case for cache layer.
type cacheClient struct{}
func (*cacheClient) Contains(ctx context.Context, key string) bool {
return cache.Default().Contains(ctx, key)
return cache.LayerCache().Contains(ctx, key)
}
func (*cacheClient) Delete(ctx context.Context, key string) error {
return cache.Default().Delete(ctx, key)
return cache.LayerCache().Delete(ctx, key)
}
func (*cacheClient) Fetch(ctx context.Context, key string, value interface{}) error {
return cache.Default().Fetch(ctx, key, value)
return cache.LayerCache().Fetch(ctx, key, value)
}
func (*cacheClient) Ping(ctx context.Context) error {
return cache.Default().Ping(ctx)
return cache.LayerCache().Ping(ctx)
}
func (*cacheClient) Save(ctx context.Context, key string, value interface{}, expiration ...time.Duration) error {
@ -57,11 +57,11 @@ func (*cacheClient) Save(ctx context.Context, key string, value interface{}, exp
return nil
}
return cache.Default().Save(ctx, key, value, expiration...)
return cache.LayerCache().Save(ctx, key, value, expiration...)
}
func (*cacheClient) Scan(ctx context.Context, match string) (cache.Iterator, error) {
return cache.Default().Scan(ctx, match)
return cache.LayerCache().Scan(ctx, match)
}
var _ Manager = &BaseManager{}