fix: support customize cache db for business (#19182)

Support to configure the customized redis db for cache layer and other
misc business for core, by default the behavior is same with
previous(stored in db 0).

Signed-off-by: chlins <chenyuzh@vmware.com>
This commit is contained in:
Chlins Zhang 2023-08-18 11:04:16 +08:00 committed by GitHub
parent 83ff2b277a
commit c7e25295fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 319 additions and 65 deletions

View File

@ -167,6 +167,28 @@ _version: 2.9.0
# max_idle_conns: 2
# max_open_conns: 0
# Uncomment redis if need to customize redis db
# redis:
# # db_index 0 is for core, it's unchangeable
# # registry_db_index: 1
# # jobservice_db_index: 2
# # trivy_db_index: 5
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_db_index: 7
# Uncomment redis if need to customize redis db
# redis:
# # db_index 0 is for core, it's unchangeable
# # registry_db_index: 1
# # jobservice_db_index: 2
# # trivy_db_index: 5
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
# Uncomment external_redis if using external Redis server
# external_redis:
# # support redis, redis+sentinel
@ -186,6 +208,10 @@ _version: 2.9.0
# jobservice_db_index: 2
# trivy_db_index: 5
# idle_timeout_seconds: 30
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
# uaa:

View File

@ -371,6 +371,49 @@ external_database:
# ssl_mode: disable
{% endif %}
{% if redis is defined %}
redis:
# # db_index 0 is for core, it's unchangeable
{% if redis.registry_db_index is defined %}
registry_db_index: {{ redis.registry_db_index }}
{% else %}
# # registry_db_index: 1
{% endif %}
{% if redis.jobservice_db_index is defined %}
jobservice_db_index: {{ redis.jobservice_db_index }}
{% else %}
# # jobservice_db_index: 2
{% endif %}
{% if redis.trivy_db_index is defined %}
trivy_db_index: {{ redis.trivy_db_index }}
{% else %}
# # trivy_db_index: 5
{% endif %}
{% if redis.harbor_db_index is defined %}
harbor_db_index: {{ redis.harbor_db_index }}
{% else %}
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
{% endif %}
{% if redis.cache_layer_db_index is defined %}
cache_layer_db_index: {{ redis.cache_layer_db_index }}
{% else %}
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% else %}
# Uncomment redis if need to customize redis db
# redis:
# # db_index 0 is for core, it's unchangeable
# # registry_db_index: 1
# # jobservice_db_index: 2
# # trivy_db_index: 5
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% if external_redis is defined %}
external_redis:
# support redis, redis+sentinel
@ -390,6 +433,18 @@ external_redis:
jobservice_db_index: {{ external_redis.jobservice_db_index }}
trivy_db_index: 5
idle_timeout_seconds: 30
{% if external_redis.harbor_db_index is defined %}
harbor_db_index: {{ redis.harbor_db_index }}
{% else %}
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
{% endif %}
{% if external_redis.cache_layer_db_index is defined %}
cache_layer_db_index: {{ redis.cache_layer_db_index }}
{% else %}
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% else %}
# Umcomments external_redis if using external Redis server
# external_redis:
@ -406,6 +461,10 @@ external_redis:
# jobservice_db_index: 2
# trivy_db_index: 5
# idle_timeout_seconds: 30
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% if uaa is defined %}

View File

@ -381,6 +381,49 @@ external_database:
# ssl_mode: disable
{% endif %}
{% if redis is defined %}
redis:
# # db_index 0 is for core, it's unchangeable
{% if redis.registry_db_index is defined %}
registry_db_index: {{ redis.registry_db_index }}
{% else %}
# # registry_db_index: 1
{% endif %}
{% if redis.jobservice_db_index is defined %}
jobservice_db_index: {{ redis.jobservice_db_index }}
{% else %}
# # jobservice_db_index: 2
{% endif %}
{% if redis.trivy_db_index is defined %}
trivy_db_index: {{ redis.trivy_db_index }}
{% else %}
# # trivy_db_index: 5
{% endif %}
{% if redis.harbor_db_index is defined %}
harbor_db_index: {{ redis.harbor_db_index }}
{% else %}
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
{% endif %}
{% if redis.cache_layer_db_index is defined %}
cache_layer_db_index: {{ redis.cache_layer_db_index }}
{% else %}
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% else %}
# Uncomment redis if need to customize redis db
# redis:
# # db_index 0 is for core, it's unchangeable
# # registry_db_index: 1
# # jobservice_db_index: 2
# # trivy_db_index: 5
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% if external_redis is defined %}
external_redis:
# support redis, redis+sentinel
@ -406,6 +449,18 @@ external_redis:
jobservice_db_index: {{ external_redis.jobservice_db_index }}
trivy_db_index: 5
idle_timeout_seconds: 30
{% if external_redis.harbor_db_index is defined %}
harbor_db_index: {{ redis.harbor_db_index }}
{% else %}
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
{% endif %}
{% if external_redis.cache_layer_db_index is defined %}
cache_layer_db_index: {{ redis.cache_layer_db_index }}
{% else %}
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% else %}
# Umcomments external_redis if using external Redis server
# external_redis:
@ -424,6 +479,10 @@ external_redis:
# jobservice_db_index: 2
# trivy_db_index: 5
# idle_timeout_seconds: 30
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% if uaa is defined %}

View File

@ -378,6 +378,49 @@ external_database:
# max_open_conns: 0
{% endif %}
{% if redis is defined %}
redis:
# # db_index 0 is for core, it's unchangeable
{% if redis.registry_db_index is defined %}
registry_db_index: {{ redis.registry_db_index }}
{% else %}
# # registry_db_index: 1
{% endif %}
{% if redis.jobservice_db_index is defined %}
jobservice_db_index: {{ redis.jobservice_db_index }}
{% else %}
# # jobservice_db_index: 2
{% endif %}
{% if redis.trivy_db_index is defined %}
trivy_db_index: {{ redis.trivy_db_index }}
{% else %}
# # trivy_db_index: 5
{% endif %}
{% if redis.harbor_db_index is defined %}
harbor_db_index: {{ redis.harbor_db_index }}
{% else %}
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
{% endif %}
{% if redis.cache_layer_db_index is defined %}
cache_layer_db_index: {{ redis.cache_layer_db_index }}
{% else %}
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% else %}
# Uncomment redis if need to customize redis db
# redis:
# # db_index 0 is for core, it's unchangeable
# # registry_db_index: 1
# # jobservice_db_index: 2
# # trivy_db_index: 5
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% if external_redis is defined %}
external_redis:
# support redis, redis+sentinel
@ -399,6 +442,18 @@ external_redis:
jobservice_db_index: {{ external_redis.jobservice_db_index }}
trivy_db_index: 5
idle_timeout_seconds: 30
{% if external_redis.harbor_db_index is defined %}
harbor_db_index: {{ redis.harbor_db_index }}
{% else %}
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
{% endif %}
{% if external_redis.cache_layer_db_index is defined %}
cache_layer_db_index: {{ redis.cache_layer_db_index }}
{% else %}
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% else %}
# Umcomments external_redis if using external Redis server
# external_redis:
@ -417,6 +472,10 @@ external_redis:
# jobservice_db_index: 2
# trivy_db_index: 5
# idle_timeout_seconds: 30
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
# # harbor_db_index: 6
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
# # cache_layer_db_index: 7
{% endif %}
{% if uaa is defined %}

View File

@ -1,6 +1,9 @@
CONFIG_PATH=/etc/core/app.conf
UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem
_REDIS_URL_CORE={{redis_url_core}}
{% if redis_url_harbor %}
_REDIS_URL_HARBOR={{redis_url_harbor}}
{% endif %}
SYNC_QUOTA=true
_REDIS_URL_REG={{redis_url_reg}}
@ -84,6 +87,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
{% endif %}
{% if cache.enabled %}
{% if redis_url_cache_layer %}
_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}}
{% endif %}
CACHE_ENABLED=true
CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
{% endif %}

View File

@ -51,6 +51,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
{% if cache.enabled %}
_REDIS_URL_CORE={{redis_url_core}}
{% if redis_url_cache_layer %}
_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}}
{% endif %}
CACHE_ENABLED=true
CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
{% endif %}

View File

@ -275,7 +275,7 @@ def parse_yaml_config(config_file_path, with_trivy):
config_dict['external_database'] = False
# update redis configs
config_dict.update(get_redis_configs(configs.get("external_redis", None), with_trivy))
config_dict.update(get_redis_configs(configs.get("redis", None), configs.get("external_redis", None), with_trivy))
# auto generated secret string for core
config_dict['core_secret'] = generate_random_string(16)
@ -371,7 +371,7 @@ def get_redis_url_param(redis=None):
return ""
def get_redis_configs(external_redis=None, with_trivy=True):
def get_redis_configs(internal_redis=None, external_redis=None, with_trivy=True):
"""Returns configs for redis
>>> get_redis_configs()['external_redis']
@ -404,6 +404,8 @@ def get_redis_configs(external_redis=None, with_trivy=True):
>>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False)
True
"""
internal_redis = internal_redis or {}
external_redis = external_redis or {}
configs = dict(external_redis=bool(external_redis))
@ -418,13 +420,22 @@ def get_redis_configs(external_redis=None, with_trivy=True):
'idle_timeout_seconds': 30,
}
# overwriting existing keys by external_redis
redis.update({key: value for (key, value) in external_redis.items() if value})
if len(internal_redis) > 0:
# overwriting existing keys by internal_redis
redis.update({key: value for (key, value) in internal_redis.items() if value})
else:
# overwriting existing keys by external_redis
redis.update({key: value for (key, value) in external_redis.items() if value})
configs['redis_url_core'] = get_redis_url(0, redis)
configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis)
configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis)
if redis.get('harbor_db_index'):
configs['redis_url_harbor'] = get_redis_url(redis['harbor_db_index'], redis)
if redis.get('cache_layer_db_index'):
configs['redis_url_cache_layer'] = get_redis_url(redis['cache_layer_db_index'], redis)
if with_trivy:
configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis)

View File

@ -262,7 +262,7 @@ func (c *controller) updateUsageByRedis(ctx context.Context, reference, referenc
return retry.Abort(ctx.Err())
}
client, err := libredis.GetCoreClient()
client, err := libredis.GetHarborClient()
if err != nil {
return retry.Abort(err)
}

View File

@ -129,25 +129,35 @@ func main() {
web.BConfig.WebConfig.Session.SessionName = config.SessionCookieName
web.BConfig.MaxMemory = 1 << 35 // (32GB)
web.BConfig.MaxUploadSize = 1 << 35 // (32GB)
redisURL := os.Getenv("_REDIS_URL_CORE")
if len(redisURL) > 0 {
u, err := url.Parse(redisURL)
// the core db used for beego session
redisCoreURL := os.Getenv("_REDIS_URL_CORE")
if len(redisCoreURL) > 0 {
_, err := url.Parse(redisCoreURL)
if err != nil {
panic("bad _REDIS_URL")
panic("bad _REDIS_URL_CORE")
}
// configure the beego session redis
web.BConfig.WebConfig.Session.SessionProvider = session.HarborProviderName
web.BConfig.WebConfig.Session.SessionProviderConfig = redisURL
log.Info("initializing cache ...")
if err := cache.Initialize(u.Scheme, redisURL); err != nil {
log.Fatalf("failed to initialize cache: %v", err)
}
// when config/db init function is called, the cache is not ready,
// enable config cache explicitly when the cache is ready
dbCfg.EnableConfigCache()
web.BConfig.WebConfig.Session.SessionProviderConfig = redisCoreURL
}
log.Info("initializing cache ...")
// the harbor db used for harbor business, use core db if not specified
redisHarborURL := os.Getenv("_REDIS_URL_HARBOR")
if redisHarborURL == "" {
redisHarborURL = redisCoreURL
}
u, err := url.Parse(redisHarborURL)
if err != nil {
panic("bad _REDIS_URL_HARBOR")
}
if err := cache.Initialize(u.Scheme, redisHarborURL); err != nil {
log.Fatalf("failed to initialize cache: %v", err)
}
// when config/db init function is called, the cache is not ready,
// enable config cache explicitly when the cache is ready
dbCfg.EnableConfigCache()
web.AddTemplateExt("htm")
log.Info("initializing configurations...")

View File

@ -19,8 +19,6 @@ import (
"errors"
"flag"
"fmt"
"net/url"
"os"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/jobservice/common/utils"
@ -29,7 +27,6 @@ import (
"github.com/goharbor/harbor/src/jobservice/job/impl"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/jobservice/runtime"
"github.com/goharbor/harbor/src/lib/cache"
cfgLib "github.com/goharbor/harbor/src/lib/config"
tracelib "github.com/goharbor/harbor/src/lib/trace"
_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
@ -47,21 +44,6 @@ func main() {
panic(fmt.Sprintf("failed to load configuration, error: %v", err))
}
// init cache if cache layer enabled
// gc needs to delete artifact by artifact manager, but the artifact cache store in
// core redis db so here require core redis url and init default cache.
if cfgLib.CacheEnabled() {
cacheURL := os.Getenv("_REDIS_URL_CORE")
u, err := url.Parse(cacheURL)
if err != nil {
panic("bad _REDIS_URL_CORE")
}
if err = cache.Initialize(u.Scheme, cacheURL); err != nil {
panic(fmt.Sprintf("failed to initialize cache: %v", err))
}
}
// Get parameters
configPath := flag.String("c", "", "Specify the yaml config file path")
flag.Parse()

View File

@ -19,6 +19,7 @@ import (
"errors"
"fmt"
"net/url"
"os"
"sync"
"time"
@ -137,3 +138,37 @@ func Initialize(typ, addr string) error {
func Default() Cache {
return cache
}
var (
// cacheLayer is the global cache layer cache instance.
cacheLayer Cache
// cacheLayerOnce is the once condition for initializing instance.
cacheLayerOnce sync.Once
)
// LayerCache is the global cache instance for cache layer.
func LayerCache() Cache {
// parse the redis url for cache layer, use the default cache if not specify
redisCacheURL := os.Getenv("_REDIS_URL_CACHE_LAYER")
if redisCacheURL == "" {
if cache != nil {
return cache
}
// use the core url if cache layer url not found
redisCacheURL = os.Getenv("_REDIS_URL_CORE")
}
u, err := url.Parse(redisCacheURL)
if err != nil {
log.Fatal("failed to parse the redis url for cache layer, bad _REDIS_URL_CACHE_LAYER")
}
cacheLayerOnce.Do(func() {
cacheLayer, err = New(u.Scheme, Address(redisCacheURL), Prefix("cache:"))
if err != nil {
log.Fatalf("failed to initialize cache for cache layer, err: %v", err)
}
})
return cacheLayer
}

View File

@ -31,9 +31,9 @@ var (
registry *redis.Client
registryOnce = &sync.Once{}
// core is a global redis client for core db
core *redis.Client
coreOnce = &sync.Once{}
// harbor is a global redis client for harbor db
harbor *redis.Client
harborOnce = &sync.Once{}
)
// GetRegistryClient returns the registry redis client.
@ -60,26 +60,30 @@ func GetRegistryClient() (*redis.Client, error) {
return registry, nil
}
// GetCoreClient returns the core redis client.
func GetCoreClient() (*redis.Client, error) {
coreOnce.Do(func() {
url := os.Getenv("_REDIS_URL_CORE")
// GetHarborClient returns the harbor redis client.
func GetHarborClient() (*redis.Client, error) {
harborOnce.Do(func() {
// parse redis url for harbor business, use core url by default
url := os.Getenv("_REDIS_URL_HARBOR")
if url == "" {
url = os.Getenv("_REDIS_URL_CORE")
}
c, err := libredis.New(cache.Options{Address: url})
if err != nil {
log.Errorf("failed to initialize redis client for core, error: %v", err)
log.Errorf("failed to initialize redis client for harbor, error: %v", err)
// reset the once to support retry if error occurred
coreOnce = &sync.Once{}
harborOnce = &sync.Once{}
return
}
if c != nil {
core = c.(*libredis.Cache).Client
harbor = c.(*libredis.Cache).Client
}
})
if core == nil {
return nil, errors.New("no core redis client initialized")
if harbor == nil {
return nil, errors.New("no harbor redis client initialized")
}
return core, nil
return harbor, nil
}

View File

@ -41,22 +41,22 @@ func TestGetRegistryClient(t *testing.T) {
}
}
func TestGetCoreClient(t *testing.T) {
func TestGetHarborClient(t *testing.T) {
// failure case with invalid address
t.Setenv("_REDIS_URL_CORE", "invalid-address")
client, err := GetCoreClient()
t.Setenv("_REDIS_URL_HARBOR", "invalid-address")
client, err := GetHarborClient()
assert.Error(t, err)
assert.Nil(t, client)
// normal case with valid address
t.Setenv("_REDIS_URL_CORE", "redis://localhost:6379/0")
client, err = GetCoreClient()
t.Setenv("_REDIS_URL_HARBOR", "redis://localhost:6379/0")
client, err = GetHarborClient()
assert.NoError(t, err)
assert.NotNil(t, client)
// multiple calls should return the same client
for i := 0; i < 10; i++ {
newClient, err := GetCoreClient()
newClient, err := GetHarborClient()
assert.NoError(t, err)
assert.Equal(t, client, newClient)
}

View File

@ -24,27 +24,27 @@ import (
)
// innerCache is the default cache client,
// actually it is a wrapper for cache.Default().
// actually it is a wrapper for cache.LayerCache().
var innerCache cache.Cache = &cacheClient{}
// cacheClient is a interceptor for cache.Default, in order to implement specific
// cacheClient is a interceptor for cache.CacheLayer, in order to implement specific
// case for cache layer.
type cacheClient struct{}
func (*cacheClient) Contains(ctx context.Context, key string) bool {
return cache.Default().Contains(ctx, key)
return cache.LayerCache().Contains(ctx, key)
}
func (*cacheClient) Delete(ctx context.Context, key string) error {
return cache.Default().Delete(ctx, key)
return cache.LayerCache().Delete(ctx, key)
}
func (*cacheClient) Fetch(ctx context.Context, key string, value interface{}) error {
return cache.Default().Fetch(ctx, key, value)
return cache.LayerCache().Fetch(ctx, key, value)
}
func (*cacheClient) Ping(ctx context.Context) error {
return cache.Default().Ping(ctx)
return cache.LayerCache().Ping(ctx)
}
func (*cacheClient) Save(ctx context.Context, key string, value interface{}, expiration ...time.Duration) error {
@ -57,11 +57,11 @@ func (*cacheClient) Save(ctx context.Context, key string, value interface{}, exp
return nil
}
return cache.Default().Save(ctx, key, value, expiration...)
return cache.LayerCache().Save(ctx, key, value, expiration...)
}
func (*cacheClient) Scan(ctx context.Context, match string) (cache.Iterator, error) {
return cache.Default().Scan(ctx, match)
return cache.LayerCache().Scan(ctx, match)
}
var _ Manager = &BaseManager{}