From 7b106d06c58de0d7e58a03f7654241c74ff4702a Mon Sep 17 00:00:00 2001 From: Steven Zou Date: Mon, 5 Nov 2018 18:23:31 +0800 Subject: [PATCH] Build logger framework to support configurable loggers/sweepers/getters Signed-off-by: Steven Zou --- src/common/utils/log/logger.go | 2 +- src/jobservice/config.yml | 22 +- src/jobservice/config/config.go | 117 +++------ src/jobservice/config/config_test.go | 107 ++++----- src/jobservice/config_test.yml | 26 +- src/jobservice/core/controller.go | 12 +- src/jobservice/core/controller_test.go | 8 +- src/jobservice/job/impl/context.go | 45 +++- src/jobservice/job/impl/default_context.go | 13 +- .../job/impl/default_context_test.go | 16 +- src/jobservice/job/impl/logger/job_logger.go | 128 ---------- .../job/impl/logger/service_logger.go | 88 ------- src/jobservice/logger/backend/file_logger.go | 90 +++++++ src/jobservice/logger/backend/std_logger.go | 84 +++++++ src/jobservice/logger/backend/utils.go | 28 +++ src/jobservice/logger/base.go | 224 ++++++++++++++++++ src/jobservice/logger/entry.go | 84 +++++++ src/jobservice/logger/factory.go | 54 +++++ src/jobservice/logger/getter/Interface.go | 12 + src/jobservice/logger/getter/file_getter.go | 37 +++ src/jobservice/logger/getter_factory.go | 27 +++ src/jobservice/logger/known_loggers.go | 81 +++++++ src/jobservice/logger/log_data_handler.go | 23 ++ src/jobservice/logger/options.go | 99 ++++++++ .../logger/{service_logger.go => service.go} | 49 ++-- src/jobservice/logger/service_logger_test.go | 87 ------- src/jobservice/logger/sweeper.go | 103 -------- src/jobservice/logger/sweeper/file_sweeper.go | 70 ++++++ src/jobservice/logger/sweeper/interface.go | 13 + src/jobservice/logger/sweeper_controller.go | 98 ++++++++ src/jobservice/logger/sweeper_factory.go | 32 +++ src/jobservice/logger/sweeper_test.go | 48 ---- src/jobservice/main.go | 30 +-- src/jobservice/runtime/bootstrap.go | 10 +- 34 files changed, 1270 insertions(+), 697 deletions(-) delete mode 100644 src/jobservice/job/impl/logger/job_logger.go delete mode 100644 src/jobservice/job/impl/logger/service_logger.go create mode 100644 src/jobservice/logger/backend/file_logger.go create mode 100644 src/jobservice/logger/backend/std_logger.go create mode 100644 src/jobservice/logger/backend/utils.go create mode 100644 src/jobservice/logger/base.go create mode 100644 src/jobservice/logger/entry.go create mode 100644 src/jobservice/logger/factory.go create mode 100644 src/jobservice/logger/getter/Interface.go create mode 100644 src/jobservice/logger/getter/file_getter.go create mode 100644 src/jobservice/logger/getter_factory.go create mode 100644 src/jobservice/logger/known_loggers.go create mode 100644 src/jobservice/logger/log_data_handler.go create mode 100644 src/jobservice/logger/options.go rename src/jobservice/logger/{service_logger.go => service.go} (69%) delete mode 100644 src/jobservice/logger/service_logger_test.go delete mode 100644 src/jobservice/logger/sweeper.go create mode 100644 src/jobservice/logger/sweeper/file_sweeper.go create mode 100644 src/jobservice/logger/sweeper/interface.go create mode 100644 src/jobservice/logger/sweeper_controller.go create mode 100644 src/jobservice/logger/sweeper_factory.go delete mode 100644 src/jobservice/logger/sweeper_test.go diff --git a/src/common/utils/log/logger.go b/src/common/utils/log/logger.go index 3fb87b4f1..06cf286a0 100644 --- a/src/common/utils/log/logger.go +++ b/src/common/utils/log/logger.go @@ -60,7 +60,7 @@ func New(out io.Writer, fmtter Formatter, lvl Level) *Logger { out: out, fmtter: fmtter, lvl: lvl, - callDepth: 3, + callDepth: 6, } } diff --git a/src/jobservice/config.yml b/src/jobservice/config.yml index 5fca1c4c8..28edfd0b8 100644 --- a/src/jobservice/config.yml +++ b/src/jobservice/config.yml @@ -22,11 +22,23 @@ worker_pool: redis_url: "localhost:6379" namespace: "harbor_job_service" -#Logger for job -logger: - path: "~/tmp/job_logs" - level: "DEBUG" - archive_period: 1 #days +#Loggers for the running job +job_loggers: + - name: "STD_OUTPUT" # logger backend name, only support "FILE" and "STD_OUTPUT" + level: "DEBUG" # INFO/DEBUG/WARNING/ERROR/FATAL + - name: "FILE" + level: "DEBUG" + settings: # Customized settings of logger + base_dir: "/Users/szou/tmp/job_logs" + sweeper: + duration: 1 #days + settings: # Customized settings of sweeper + work_dir: "/Users/szou/tmp/job_logs" + +#Loggers for the job service +loggers: + - name: "STD_OUTPUT" # Same with above + level: "DEBUG" #Admin server endpoint admin_server: "http://adminserver:9010/" diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index a44724631..34be52e5a 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -28,19 +28,16 @@ import ( ) const ( - jobServiceProtocol = "JOB_SERVICE_PROTOCOL" - jobServicePort = "JOB_SERVICE_PORT" - jobServiceHTTPCert = "JOB_SERVICE_HTTPS_CERT" - jobServiceHTTPKey = "JOB_SERVICE_HTTPS_KEY" - jobServiceWorkerPoolBackend = "JOB_SERVICE_POOL_BACKEND" - jobServiceWorkers = "JOB_SERVICE_POOL_WORKERS" - jobServiceRedisURL = "JOB_SERVICE_POOL_REDIS_URL" - jobServiceRedisNamespace = "JOB_SERVICE_POOL_REDIS_NAMESPACE" - jobServiceLoggerBasePath = "JOB_SERVICE_LOGGER_BASE_PATH" - jobServiceLoggerLevel = "JOB_SERVICE_LOGGER_LEVEL" - jobServiceLoggerArchivePeriod = "JOB_SERVICE_LOGGER_ARCHIVE_PERIOD" - jobServiceCoreServerEndpoint = "CORE_URL" - jobServiceAuthSecret = "JOBSERVICE_SECRET" + jobServiceProtocol = "JOB_SERVICE_PROTOCOL" + jobServicePort = "JOB_SERVICE_PORT" + jobServiceHTTPCert = "JOB_SERVICE_HTTPS_CERT" + jobServiceHTTPKey = "JOB_SERVICE_HTTPS_KEY" + jobServiceWorkerPoolBackend = "JOB_SERVICE_POOL_BACKEND" + jobServiceWorkers = "JOB_SERVICE_POOL_WORKERS" + jobServiceRedisURL = "JOB_SERVICE_POOL_REDIS_URL" + jobServiceRedisNamespace = "JOB_SERVICE_POOL_REDIS_NAMESPACE" + jobServiceCoreServerEndpoint = "CORE_URL" + jobServiceAuthSecret = "JOBSERVICE_SECRET" // JobServiceProtocolHTTPS points to the 'https' protocol JobServiceProtocolHTTPS = "https" @@ -76,8 +73,11 @@ type Configuration struct { // Configurations of worker pool PoolConfig *PoolConfig `yaml:"worker_pool,omitempty"` + // Job logger configurations + JobLoggerConfigs []*LoggerConfig `yaml:"job_loggers,omitempty"` + // Logger configurations - LoggerConfig *LoggerConfig `yaml:"logger,omitempty"` + LoggerConfigs []*LoggerConfig `yaml:"loggers,omitempty"` } // HTTPSConfig keeps additional configurations when using https protocol @@ -100,11 +100,21 @@ type PoolConfig struct { RedisPoolCfg *RedisPoolConfig `yaml:"redis_pool,omitempty"` } -// LoggerConfig keeps logger configurations. +// CustomizedSettings keeps the customized settings of logger +type CustomizedSettings map[string]interface{} + +// LogSweeperConfig keeps settings of log sweeper +type LogSweeperConfig struct { + Duration int `yaml:"duration"` + Settings CustomizedSettings `yaml:"settings"` +} + +// LoggerConfig keeps logger basic configurations. type LoggerConfig struct { - BasePath string `yaml:"path"` - LogLevel string `yaml:"level"` - ArchivePeriod uint `yaml:"archive_period"` + Name string `yaml:"name"` + Level string `yaml:"level"` + Settings CustomizedSettings `yaml:"settings"` + Sweeper *LogSweeperConfig `yaml:"sweeper"` } // Load the configuration options from the specified yaml file. @@ -151,33 +161,6 @@ func (c *Configuration) Load(yamlFilePath string, detectEnv bool) error { return c.validate() } -// GetLogBasePath returns the log base path config -func GetLogBasePath() string { - if DefaultConfig.LoggerConfig != nil { - return DefaultConfig.LoggerConfig.BasePath - } - - return "" -} - -// GetLogLevel returns the log level -func GetLogLevel() string { - if DefaultConfig.LoggerConfig != nil { - return DefaultConfig.LoggerConfig.LogLevel - } - - return "" -} - -// GetLogArchivePeriod returns the archive period -func GetLogArchivePeriod() uint { - if DefaultConfig.LoggerConfig != nil { - return DefaultConfig.LoggerConfig.ArchivePeriod - } - - return 1 // return default -} - // GetAuthSecret get the auth secret from the env func GetAuthSecret() string { return utils.ReadEnv(jobServiceAuthSecret) @@ -268,31 +251,6 @@ func (c *Configuration) loadEnvs() { } } - // logger - loggerPath := utils.ReadEnv(jobServiceLoggerBasePath) - if !utils.IsEmptyStr(loggerPath) { - if c.LoggerConfig == nil { - c.LoggerConfig = &LoggerConfig{} - } - c.LoggerConfig.BasePath = loggerPath - } - loggerLevel := utils.ReadEnv(jobServiceLoggerLevel) - if !utils.IsEmptyStr(loggerLevel) { - if c.LoggerConfig == nil { - c.LoggerConfig = &LoggerConfig{} - } - c.LoggerConfig.LogLevel = loggerLevel - } - archivePeriod := utils.ReadEnv(jobServiceLoggerArchivePeriod) - if !utils.IsEmptyStr(archivePeriod) { - if period, err := strconv.Atoi(archivePeriod); err == nil { - if c.LoggerConfig == nil { - c.LoggerConfig = &LoggerConfig{} - } - c.LoggerConfig.ArchivePeriod = uint(period) - } - } - // admin server if coreServer := utils.ReadEnv(jobServiceCoreServerEndpoint); !utils.IsEmptyStr(coreServer) { c.AdminServer = coreServer @@ -357,21 +315,14 @@ func (c *Configuration) validate() error { } } - if c.LoggerConfig == nil { - return errors.New("missing logger config") + // Job service loggers + if len(c.LoggerConfigs) == 0 { + return errors.New("missing logger config of job service") } - if !utils.DirExists(c.LoggerConfig.BasePath) { - return errors.New("logger path should be an existing dir") - } - - validLevels := "DEBUG,INFO,WARNING,ERROR,FATAL" - if !strings.Contains(validLevels, c.LoggerConfig.LogLevel) { - return fmt.Errorf("logger level can only be one of: %s", validLevels) - } - - if c.LoggerConfig.ArchivePeriod == 0 { - return fmt.Errorf("logger archive period should be greater than 0") + // Job loggers + if len(c.JobLoggerConfigs) == 0 { + return errors.New("missing logger config of job") } if _, err := url.Parse(c.AdminServer); err != nil { diff --git a/src/jobservice/config/config_test.go b/src/jobservice/config/config_test.go index 5d841681c..687f1d8ab 100644 --- a/src/jobservice/config/config_test.go +++ b/src/jobservice/config/config_test.go @@ -26,24 +26,13 @@ func TestConfigLoadingFailed(t *testing.T) { } func TestConfigLoadingSucceed(t *testing.T) { - if err := CreateLogDir(); err != nil { - t.Fatal(err) - } - cfg := &Configuration{} if err := cfg.Load("../config_test.yml", false); err != nil { t.Fatalf("Load config from yaml file, expect nil error but got error '%s'\n", err) } - - if err := RemoveLogDir(); err != nil { - t.Fatal(err) - } } func TestConfigLoadingWithEnv(t *testing.T) { - if err := CreateLogDir(); err != nil { - t.Error(err) - } setENV() cfg := &Configuration{} @@ -52,68 +41,74 @@ func TestConfigLoadingWithEnv(t *testing.T) { } if cfg.Protocol != "https" { - t.Fatalf("expect protocol 'https', but got '%s'\n", cfg.Protocol) + t.Errorf("expect protocol 'https', but got '%s'\n", cfg.Protocol) } if cfg.Port != 8989 { - t.Fatalf("expect port 8989 but got '%d'\n", cfg.Port) + t.Errorf("expect port 8989 but got '%d'\n", cfg.Port) } if cfg.PoolConfig.WorkerCount != 8 { - t.Fatalf("expect workcount 8 but go '%d'\n", cfg.PoolConfig.WorkerCount) + t.Errorf("expect workcount 8 but go '%d'\n", cfg.PoolConfig.WorkerCount) } if cfg.PoolConfig.RedisPoolCfg.RedisURL != "redis://arbitrary_username:password@8.8.8.8:6379/0" { - t.Fatalf("expect redis URL 'localhost' but got '%s'\n", cfg.PoolConfig.RedisPoolCfg.RedisURL) + t.Errorf("expect redis URL 'localhost' but got '%s'\n", cfg.PoolConfig.RedisPoolCfg.RedisURL) } if cfg.PoolConfig.RedisPoolCfg.Namespace != "ut_namespace" { - t.Fatalf("expect redis namespace 'ut_namespace' but got '%s'\n", cfg.PoolConfig.RedisPoolCfg.Namespace) + t.Errorf("expect redis namespace 'ut_namespace' but got '%s'\n", cfg.PoolConfig.RedisPoolCfg.Namespace) } - if cfg.LoggerConfig.BasePath != "/tmp" { - t.Fatalf("expect log base path '/tmp' but got '%s'\n", cfg.LoggerConfig.BasePath) + if GetAuthSecret() != "js_secret" { + t.Errorf("expect auth secret 'js_secret' but got '%s'", GetAuthSecret()) } - if cfg.LoggerConfig.LogLevel != "DEBUG" { - t.Fatalf("expect log level 'DEBUG' but got '%s'\n", cfg.LoggerConfig.LogLevel) - } - if cfg.LoggerConfig.ArchivePeriod != 5 { - t.Fatalf("expect log archive period 5 but got '%d'\n", cfg.LoggerConfig.ArchivePeriod) + if GetUIAuthSecret() != "core_secret" { + t.Errorf("expect auth secret 'core_secret' but got '%s'", GetUIAuthSecret()) } unsetENV() - if err := RemoveLogDir(); err != nil { - t.Fatal(err) - } } func TestDefaultConfig(t *testing.T) { - if err := CreateLogDir(); err != nil { - t.Fatal(err) - } - if err := DefaultConfig.Load("../config_test.yml", true); err != nil { t.Fatalf("Load config from yaml file, expect nil error but got error '%s'\n", err) } if endpoint := GetAdminServerEndpoint(); endpoint != "http://127.0.0.1:8888" { - t.Fatalf("expect default admin server endpoint 'http://127.0.0.1:8888' but got '%s'\n", endpoint) - } - - if basePath := GetLogBasePath(); basePath != "/tmp/job_logs" { - t.Fatalf("expect default logger base path '/tmp/job_logs' but got '%s'\n", basePath) - } - - if lvl := GetLogLevel(); lvl != "INFO" { - t.Fatalf("expect default logger level 'INFO' but got '%s'\n", lvl) - } - - if period := GetLogArchivePeriod(); period != 1 { - t.Fatalf("expect default log archive period 1 but got '%d'\n", period) + t.Errorf("expect default admin server endpoint 'http://127.0.0.1:8888' but got '%s'\n", endpoint) } redisURL := DefaultConfig.PoolConfig.RedisPoolCfg.RedisURL - if redisURL != "redis://redis:6379" { - t.Fatalf("expect redisURL '%s' but got '%s'\n", "redis://redis:6379", redisURL) + if redisURL != "redis://localhost:6379" { + t.Errorf("expect redisURL '%s' but got '%s'\n", "redis://localhost:6379", redisURL) } - if err := RemoveLogDir(); err != nil { - t.Fatal(err) + if len(DefaultConfig.JobLoggerConfigs) == 0 { + t.Errorf("expect 2 job loggers configured but got %d", len(DefaultConfig.JobLoggerConfigs)) + } + + if len(DefaultConfig.LoggerConfigs) == 0 { + t.Errorf("expect 1 loggers configured but got %d", len(DefaultConfig.LoggerConfigs)) + } + + // Only verify the complicated one + theLogger := DefaultConfig.JobLoggerConfigs[1] + if theLogger.Name != "FILE" { + t.Fatalf("expect FILE logger but got %s", theLogger.Name) + } + if theLogger.Level != "INFO" { + t.Errorf("expect INFO log level of FILE logger but got %s", theLogger.Level) + } + if len(theLogger.Settings) == 0 { + t.Errorf("expect extra settings but got nothing") + } + if theLogger.Settings["base_dir"] != "/tmp/job_logs" { + t.Errorf("expect extra setting base_dir to be '/tmp/job_logs' but got %s", theLogger.Settings["base_dir"]) + } + if theLogger.Sweeper == nil { + t.Fatalf("expect non nil sweeper of FILE logger but got nil") + } + if theLogger.Sweeper.Duration != 5 { + t.Errorf("expect sweep duration to be 5 but got %d", theLogger.Sweeper.Duration) + } + if theLogger.Sweeper.Settings["work_dir"] != "/tmp/job_logs" { + t.Errorf("expect work dir of sweeper of FILE logger to be '/tmp/job_logs' but got %s", theLogger.Sweeper.Settings["work_dir"]) } } @@ -126,9 +121,8 @@ func setENV() { os.Setenv("JOB_SERVICE_POOL_WORKERS", "8") os.Setenv("JOB_SERVICE_POOL_REDIS_URL", "8.8.8.8:6379,100,password,0") os.Setenv("JOB_SERVICE_POOL_REDIS_NAMESPACE", "ut_namespace") - os.Setenv("JOB_SERVICE_LOGGER_BASE_PATH", "/tmp") - os.Setenv("JOB_SERVICE_LOGGER_LEVEL", "DEBUG") - os.Setenv("JOB_SERVICE_LOGGER_ARCHIVE_PERIOD", "5") + os.Setenv("JOBSERVICE_SECRET", "js_secret") + os.Setenv("CORE_SECRET", "core_secret") } func unsetENV() { @@ -140,15 +134,6 @@ func unsetENV() { os.Unsetenv("JOB_SERVICE_POOL_WORKERS") os.Unsetenv("JOB_SERVICE_POOL_REDIS_URL") os.Unsetenv("JOB_SERVICE_POOL_REDIS_NAMESPACE") - os.Unsetenv("JOB_SERVICE_LOGGER_BASE_PATH") - os.Unsetenv("JOB_SERVICE_LOGGER_LEVEL") - os.Unsetenv("JOB_SERVICE_LOGGER_ARCHIVE_PERIOD") -} - -func CreateLogDir() error { - return os.MkdirAll("/tmp/job_logs", 0755) -} - -func RemoveLogDir() error { - return os.Remove("/tmp/job_logs") + os.Unsetenv("JOBSERVICE_SECRET") + os.Unsetenv("CORE_SECRET") } diff --git a/src/jobservice/config_test.yml b/src/jobservice/config_test.yml index 47a19fae3..057a35934 100644 --- a/src/jobservice/config_test.yml +++ b/src/jobservice/config_test.yml @@ -8,7 +8,7 @@ https_config: key: "../server.key" #Server listening port -port: 9443 +port: 9444 #Worker pool worker_pool: @@ -19,14 +19,26 @@ worker_pool: redis_pool: #redis://[arbitrary_username:password@]ipaddress:port/database_index #or ipaddress:port[,weight,password,database_index] - redis_url: "redis:6379" + redis_url: "localhost:6379" namespace: "harbor_job_service" -#Logger for job -logger: - path: "/tmp/job_logs" - level: "INFO" - archive_period: 1 #days +#Loggers for the running job +job_loggers: + - name: "STD_OUTPUT" # logger backend name, only support "FILE" and "STD_OUTPUT" + level: "DEBUG" # INFO/DEBUG/WARNING/ERROR/FATAL + - name: "FILE" + level: "INFO" + settings: # Customized settings of logger + base_dir: "/tmp/job_logs" + sweeper: + duration: 5 #days + settings: # Customized settings of sweeper + work_dir: "/tmp/job_logs" + +#Loggers for the job service +loggers: + - name: "STD_OUTPUT" # Same with above + level: "DEBUG" #Admin server endpoint admin_server: "http://127.0.0.1:8888" diff --git a/src/jobservice/core/controller.go b/src/jobservice/core/controller.go index 9badf1a03..71fce9d42 100644 --- a/src/jobservice/core/controller.go +++ b/src/jobservice/core/controller.go @@ -17,10 +17,9 @@ package core import ( "errors" "fmt" - "io/ioutil" - "github.com/goharbor/harbor/src/jobservice/config" - "github.com/goharbor/harbor/src/jobservice/errs" + "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/jobservice/models" "github.com/goharbor/harbor/src/jobservice/pool" @@ -141,12 +140,7 @@ func (c *Controller) GetJobLogData(jobID string) ([]byte, error) { return nil, errors.New("empty job ID") } - logPath := fmt.Sprintf("%s/%s.log", config.GetLogBasePath(), jobID) - if !utils.FileExists(logPath) { - return nil, errs.NoObjectFoundError(fmt.Sprintf("%s.log", jobID)) - } - - logData, err := ioutil.ReadFile(logPath) + logData, err := logger.Retrieve(jobID) if err != nil { return nil, err } diff --git a/src/jobservice/core/controller_test.go b/src/jobservice/core/controller_test.go index 44c9568f0..1d9f57e89 100644 --- a/src/jobservice/core/controller_test.go +++ b/src/jobservice/core/controller_test.go @@ -17,8 +17,6 @@ import ( "errors" "testing" - "github.com/goharbor/harbor/src/jobservice/errs" - "github.com/goharbor/harbor/src/jobservice/env" "github.com/goharbor/harbor/src/jobservice/models" ) @@ -141,11 +139,7 @@ func TestGetJobLogData(t *testing.T) { pool := &fakePool{} c := NewController(pool) - if _, err := c.GetJobLogData("fake_ID"); err != nil { - if !errs.IsObjectNotFoundError(err) { - t.Errorf("expect object not found error but got '%s'\n", err) - } - } else { + if _, err := c.GetJobLogData("fake_ID"); err == nil { t.Fatal("expect error but got nil") } } diff --git a/src/jobservice/job/impl/context.go b/src/jobservice/job/impl/context.go index a7ce8e1ae..2be39a3dd 100644 --- a/src/jobservice/job/impl/context.go +++ b/src/jobservice/job/impl/context.go @@ -29,7 +29,6 @@ import ( "github.com/goharbor/harbor/src/jobservice/config" "github.com/goharbor/harbor/src/jobservice/env" "github.com/goharbor/harbor/src/jobservice/job" - jlogger "github.com/goharbor/harbor/src/jobservice/job/impl/logger" "github.com/goharbor/harbor/src/jobservice/logger" jmodel "github.com/goharbor/harbor/src/jobservice/models" ) @@ -124,11 +123,11 @@ func (c *Context) Build(dep env.JobData) (env.JobContext, error) { jContext.properties[k] = v } - // Init logger here - logPath := fmt.Sprintf("%s/%s.log", config.GetLogBasePath(), dep.ID) - jContext.logger = jlogger.New(logPath, config.GetLogLevel()) - if jContext.logger == nil { - return nil, errors.New("failed to initialize job logger") + // Set loggers for job + if err := setLoggers(func(lg logger.Interface) { + jContext.logger = lg + }, dep.ID); err != nil { + return nil, err } if opCommandFunc, ok := dep.ExtraData["opCommandFunc"]; ok { @@ -227,3 +226,37 @@ func getDBFromConfig(cfg map[string]interface{}) *models.Database { return database } + +// create loggers based on the configurations and set it to the job executing context. +func setLoggers(setter func(lg logger.Interface), jobID string) error { + if setter == nil { + return errors.New("missing setter func") + } + + // Init job loggers here + lOptions := []logger.Option{} + for _, lc := range config.DefaultConfig.JobLoggerConfigs { + if lc.Name == logger.LoggerNameFile { + // Need extra param + fSettings := map[string]interface{}{} + for k, v := range lc.Settings { + // Copy settings + fSettings[k] = v + } + // Append file name param + fSettings["filename"] = fmt.Sprintf("%s.log", jobID) + lOptions = append(lOptions, logger.BackendOption(lc.Name, lc.Level, fSettings)) + } else { + lOptions = append(lOptions, logger.BackendOption(lc.Name, lc.Level, lc.Settings)) + } + } + // Get logger for the job + lg, err := logger.GetLogger(lOptions...) + if err != nil { + return fmt.Errorf("initialize job logger error: %s", err) + } + + setter(lg) + + return nil +} diff --git a/src/jobservice/job/impl/default_context.go b/src/jobservice/job/impl/default_context.go index 7231f5446..34243059b 100644 --- a/src/jobservice/job/impl/default_context.go +++ b/src/jobservice/job/impl/default_context.go @@ -17,13 +17,10 @@ package impl import ( "context" "errors" - "fmt" "reflect" - "github.com/goharbor/harbor/src/jobservice/config" "github.com/goharbor/harbor/src/jobservice/env" "github.com/goharbor/harbor/src/jobservice/job" - jlogger "github.com/goharbor/harbor/src/jobservice/job/impl/logger" "github.com/goharbor/harbor/src/jobservice/logger" jmodel "github.com/goharbor/harbor/src/jobservice/models" ) @@ -72,11 +69,11 @@ func (c *DefaultContext) Build(dep env.JobData) (env.JobContext, error) { } } - // Init logger here - logPath := fmt.Sprintf("%s/%s.log", config.GetLogBasePath(), dep.ID) - jContext.logger = jlogger.New(logPath, config.GetLogLevel()) - if jContext.logger == nil { - return nil, errors.New("failed to initialize job logger") + // Set loggers for job + if err := setLoggers(func(lg logger.Interface) { + jContext.logger = lg + }, dep.ID); err != nil { + return nil, err } if opCommandFunc, ok := dep.ExtraData["opCommandFunc"]; ok { diff --git a/src/jobservice/job/impl/default_context_test.go b/src/jobservice/job/impl/default_context_test.go index e3282587d..c37d63d0a 100644 --- a/src/jobservice/job/impl/default_context_test.go +++ b/src/jobservice/job/impl/default_context_test.go @@ -57,15 +57,19 @@ func TestDefaultContext(t *testing.T) { jobData.ExtraData["checkInFunc"] = checkInFunc jobData.ExtraData["launchJobFunc"] = launchJobFunc - oldLogConfig := config.DefaultConfig.LoggerConfig + oldLogConfig := config.DefaultConfig.JobLoggerConfigs defer func() { - config.DefaultConfig.LoggerConfig = oldLogConfig + config.DefaultConfig.JobLoggerConfigs = oldLogConfig }() - config.DefaultConfig.LoggerConfig = &config.LoggerConfig{ - LogLevel: "debug", - ArchivePeriod: 1, - BasePath: os.TempDir(), + logSettings := map[string]interface{}{} + logSettings["base_dir"] = os.TempDir() + config.DefaultConfig.JobLoggerConfigs = []*config.LoggerConfig{ + { + Level: "DEBUG", + Name: "FILE", + Settings: logSettings, + }, } newJobContext, err := defaultContext.Build(jobData) diff --git a/src/jobservice/job/impl/logger/job_logger.go b/src/jobservice/job/impl/logger/job_logger.go deleted file mode 100644 index 2238e71d2..000000000 --- a/src/jobservice/job/impl/logger/job_logger.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logger - -import ( - "os" - "strings" - - "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/jobservice/logger" -) - -// JobLogger is an implementation of logger.Interface. -// It used in the job to output logs to the logfile. -type JobLogger struct { - backendLogger *log.Logger - streamRef *os.File -} - -// New logger -// nil might be returned -func New(logPath string, level string) logger.Interface { - f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - logger.Errorf("Failed to create job logger: %s", err) - return nil - } - logLevel := parseLevel(level) - backendLogger := log.New(f, log.NewTextFormatter(), logLevel) - - return &JobLogger{ - backendLogger: backendLogger, - streamRef: f, - } -} - -// Close the opened io stream -// Implements logger.Closer interface -func (jl *JobLogger) Close() error { - if jl.streamRef != nil { - return jl.streamRef.Close() - } - - return nil -} - -// Debug ... -func (jl *JobLogger) Debug(v ...interface{}) { - jl.backendLogger.Debug(v...) -} - -// Debugf with format -func (jl *JobLogger) Debugf(format string, v ...interface{}) { - jl.backendLogger.Debugf(format, v...) -} - -// Info ... -func (jl *JobLogger) Info(v ...interface{}) { - jl.backendLogger.Info(v...) -} - -// Infof with format -func (jl *JobLogger) Infof(format string, v ...interface{}) { - jl.backendLogger.Infof(format, v...) -} - -// Warning ... -func (jl *JobLogger) Warning(v ...interface{}) { - jl.backendLogger.Warning(v...) -} - -// Warningf with format -func (jl *JobLogger) Warningf(format string, v ...interface{}) { - jl.backendLogger.Warningf(format, v...) -} - -// Error ... -func (jl *JobLogger) Error(v ...interface{}) { - jl.backendLogger.Error(v...) -} - -// Errorf with format -func (jl *JobLogger) Errorf(format string, v ...interface{}) { - jl.backendLogger.Errorf(format, v...) -} - -// Fatal error -func (jl *JobLogger) Fatal(v ...interface{}) { - jl.backendLogger.Fatal(v...) -} - -// Fatalf error -func (jl *JobLogger) Fatalf(format string, v ...interface{}) { - jl.backendLogger.Fatalf(format, v...) -} - -func parseLevel(lvl string) log.Level { - - var level = log.WarningLevel - - switch strings.ToLower(lvl) { - case "debug": - level = log.DebugLevel - case "info": - level = log.InfoLevel - case "warning": - level = log.WarningLevel - case "error": - level = log.ErrorLevel - case "fatal": - level = log.FatalLevel - default: - } - - return level -} diff --git a/src/jobservice/job/impl/logger/service_logger.go b/src/jobservice/job/impl/logger/service_logger.go deleted file mode 100644 index ac0df4184..000000000 --- a/src/jobservice/job/impl/logger/service_logger.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logger - -import ( - "os" - - "github.com/goharbor/harbor/src/common/utils/log" -) - -// ServiceLogger is an implementation of logger.Interface. -// It used to log info in workerpool components. -type ServiceLogger struct { - backendLogger *log.Logger -} - -// NewServiceLogger to create new logger for job service -// nil might be returned -func NewServiceLogger(level string) *ServiceLogger { - logLevel := parseLevel(level) - backendLogger := log.New(os.Stdout, log.NewTextFormatter(), logLevel) - - return &ServiceLogger{ - backendLogger: backendLogger, - } -} - -// Debug ... -func (sl *ServiceLogger) Debug(v ...interface{}) { - sl.backendLogger.Debug(v...) -} - -// Debugf with format -func (sl *ServiceLogger) Debugf(format string, v ...interface{}) { - sl.backendLogger.Debugf(format, v...) -} - -// Info ... -func (sl *ServiceLogger) Info(v ...interface{}) { - sl.backendLogger.Info(v...) -} - -// Infof with format -func (sl *ServiceLogger) Infof(format string, v ...interface{}) { - sl.backendLogger.Infof(format, v...) -} - -// Warning ... -func (sl *ServiceLogger) Warning(v ...interface{}) { - sl.backendLogger.Warning(v...) -} - -// Warningf with format -func (sl *ServiceLogger) Warningf(format string, v ...interface{}) { - sl.backendLogger.Warningf(format, v...) -} - -// Error ... -func (sl *ServiceLogger) Error(v ...interface{}) { - sl.backendLogger.Error(v...) -} - -// Errorf with format -func (sl *ServiceLogger) Errorf(format string, v ...interface{}) { - sl.backendLogger.Errorf(format, v...) -} - -// Fatal error -func (sl *ServiceLogger) Fatal(v ...interface{}) { - sl.backendLogger.Fatal(v...) -} - -// Fatalf error -func (sl *ServiceLogger) Fatalf(format string, v ...interface{}) { - sl.backendLogger.Fatalf(format, v...) -} diff --git a/src/jobservice/logger/backend/file_logger.go b/src/jobservice/logger/backend/file_logger.go new file mode 100644 index 000000000..065f89e51 --- /dev/null +++ b/src/jobservice/logger/backend/file_logger.go @@ -0,0 +1,90 @@ +package backend + +import ( + "os" + + "github.com/goharbor/harbor/src/common/utils/log" +) + +// FileLogger is an implementation of logger.Interface. +// It outputs logs to the specified logfile. +type FileLogger struct { + backendLogger *log.Logger + streamRef *os.File +} + +// NewFileLogger crates a new file logger +// nil might be returned +func NewFileLogger(level string, logPath string) (*FileLogger, error) { + f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return nil, err + } + logLevel := parseLevel(level) + backendLogger := log.New(f, log.NewTextFormatter(), logLevel) + + return &FileLogger{ + backendLogger: backendLogger, + streamRef: f, + }, nil +} + +// Close the opened io stream +// Implements logger.Closer interface +func (fl *FileLogger) Close() error { + if fl.streamRef != nil { + return fl.streamRef.Close() + } + + return nil +} + +// Debug ... +func (fl *FileLogger) Debug(v ...interface{}) { + fl.backendLogger.Debug(v...) +} + +// Debugf with format +func (fl *FileLogger) Debugf(format string, v ...interface{}) { + fl.backendLogger.Debugf(format, v...) +} + +// Info ... +func (fl *FileLogger) Info(v ...interface{}) { + fl.backendLogger.Info(v...) +} + +// Infof with format +func (fl *FileLogger) Infof(format string, v ...interface{}) { + fl.backendLogger.Infof(format, v...) +} + +// Warning ... +func (fl *FileLogger) Warning(v ...interface{}) { + fl.backendLogger.Warning(v...) +} + +// Warningf with format +func (fl *FileLogger) Warningf(format string, v ...interface{}) { + fl.backendLogger.Warningf(format, v...) +} + +// Error ... +func (fl *FileLogger) Error(v ...interface{}) { + fl.backendLogger.Error(v...) +} + +// Errorf with format +func (fl *FileLogger) Errorf(format string, v ...interface{}) { + fl.backendLogger.Errorf(format, v...) +} + +// Fatal error +func (fl *FileLogger) Fatal(v ...interface{}) { + fl.backendLogger.Fatal(v...) +} + +// Fatalf error +func (fl *FileLogger) Fatalf(format string, v ...interface{}) { + fl.backendLogger.Fatalf(format, v...) +} diff --git a/src/jobservice/logger/backend/std_logger.go b/src/jobservice/logger/backend/std_logger.go new file mode 100644 index 000000000..4bbe0ddc6 --- /dev/null +++ b/src/jobservice/logger/backend/std_logger.go @@ -0,0 +1,84 @@ +package backend + +import ( + "os" + + "github.com/goharbor/harbor/src/common/utils/log" +) + +const ( + // StdOut represents os.Stdout + StdOut = "std_out" + // StdErr represents os.StdErr + StdErr = "std_err" +) + +// StdOutputLogger is an implementation of logger.Interface. +// It outputs the log to the stdout/stderr. +type StdOutputLogger struct { + backendLogger *log.Logger +} + +// NewStdOutputLogger creates a new std output logger +func NewStdOutputLogger(level string, output string) *StdOutputLogger { + logLevel := parseLevel(level) + logStream := os.Stdout + if output == StdErr { + logStream = os.Stderr + } + backendLogger := log.New(logStream, log.NewTextFormatter(), logLevel) + + return &StdOutputLogger{ + backendLogger: backendLogger, + } +} + +// Debug ... +func (sl *StdOutputLogger) Debug(v ...interface{}) { + sl.backendLogger.Debug(v...) +} + +// Debugf with format +func (sl *StdOutputLogger) Debugf(format string, v ...interface{}) { + sl.backendLogger.Debugf(format, v...) +} + +// Info ... +func (sl *StdOutputLogger) Info(v ...interface{}) { + sl.backendLogger.Info(v...) +} + +// Infof with format +func (sl *StdOutputLogger) Infof(format string, v ...interface{}) { + sl.backendLogger.Infof(format, v...) +} + +// Warning ... +func (sl *StdOutputLogger) Warning(v ...interface{}) { + sl.backendLogger.Warning(v...) +} + +// Warningf with format +func (sl *StdOutputLogger) Warningf(format string, v ...interface{}) { + sl.backendLogger.Warningf(format, v...) +} + +// Error ... +func (sl *StdOutputLogger) Error(v ...interface{}) { + sl.backendLogger.Error(v...) +} + +// Errorf with format +func (sl *StdOutputLogger) Errorf(format string, v ...interface{}) { + sl.backendLogger.Errorf(format, v...) +} + +// Fatal error +func (sl *StdOutputLogger) Fatal(v ...interface{}) { + sl.backendLogger.Fatal(v...) +} + +// Fatalf error +func (sl *StdOutputLogger) Fatalf(format string, v ...interface{}) { + sl.backendLogger.Fatalf(format, v...) +} diff --git a/src/jobservice/logger/backend/utils.go b/src/jobservice/logger/backend/utils.go new file mode 100644 index 000000000..99c83427d --- /dev/null +++ b/src/jobservice/logger/backend/utils.go @@ -0,0 +1,28 @@ +package backend + +import ( + "strings" + + "github.com/goharbor/harbor/src/common/utils/log" +) + +func parseLevel(lvl string) log.Level { + + var level = log.WarningLevel + + switch strings.ToLower(lvl) { + case "debug": + level = log.DebugLevel + case "info": + level = log.InfoLevel + case "warning": + level = log.WarningLevel + case "error": + level = log.ErrorLevel + case "fatal": + level = log.FatalLevel + default: + } + + return level +} diff --git a/src/jobservice/logger/base.go b/src/jobservice/logger/base.go new file mode 100644 index 000000000..b0743ca7c --- /dev/null +++ b/src/jobservice/logger/base.go @@ -0,0 +1,224 @@ +package logger + +import ( + "context" + "errors" + "fmt" + "sort" + "sync" + + "github.com/goharbor/harbor/src/jobservice/config" + "github.com/goharbor/harbor/src/jobservice/logger/getter" + "github.com/goharbor/harbor/src/jobservice/logger/sweeper" +) + +var singletons sync.Map + +// GetLogger gets an unified logger entry for logging per the passed settings. +// The logger may built based on the multiple registered logger backends. +// +// loggerOptions ...Option : logger options +// +// If failed, a nil logger and a non-nil error will be returned. +// Otherwise, a non nil logger is returned with nil error. +func GetLogger(loggerOptions ...Option) (Interface, error) { + lOptions := &options{ + values: make(map[string][]OptionItem), + } + + // Config + for _, op := range loggerOptions { + op.Apply(lOptions) + } + + // No options specified, enable std as default + if len(loggerOptions) == 0 { + defaultOp := BackendOption(LoggerNameStdOutput, "", nil) + defaultOp.Apply(lOptions) + } + + // Create backends + loggers := []Interface{} + for name, ops := range lOptions.values { + if !IsKnownLogger(name) { + return nil, fmt.Errorf("no logger registered for name '%s'", name) + } + + d := KnownLoggers(name) + var ( + l Interface + ok bool + ) + + // Singleton + if d.Singleton { + var li interface{} + li, ok = singletons.Load(name) + if ok { + l = li.(Interface) + } + } + + if !ok { + var err error + l, err = d.Logger(ops...) + if err != nil { + return nil, err + } + + // Cache it + singletons.Store(name, l) + } + + // Append to the logger list as logger entry backends + loggers = append(loggers, l) + } + + return NewEntry(loggers), nil +} + +// GetSweeper gets an unified sweeper controller for sweeping purpose. +// +// context context.Context : system contex used to control the sweeping loops +// sweeperOptions ...Option : sweeper options +// +// If failed, a nil sweeper and a non-nil error will be returned. +// Otherwise, a non nil sweeper is returned with nil error. +func GetSweeper(context context.Context, sweeperOptions ...Option) (sweeper.Interface, error) { + // No default sweeper will provdie + // If no one is configured, directly return nil with error + if len(sweeperOptions) == 0 { + return nil, errors.New("no options provided for creating sweeper controller") + } + + sOptions := &options{ + values: make(map[string][]OptionItem), + } + + // Config + for _, op := range sweeperOptions { + op.Apply(sOptions) + } + + sweepers := []sweeper.Interface{} + for name, ops := range sOptions.values { + if !HasSweeper(name) { + return nil, fmt.Errorf("no sweeper provided for the logger %s", name) + } + + d := KnownLoggers(name) + s, err := d.Sweeper(ops...) + if err != nil { + return nil, err + } + + sweepers = append(sweepers, s) + } + + return NewSweeperController(context, sweepers), nil +} + +// GetLogDataGetter return the 1st matched log data getter interface +// +// loggerOptions ...Option : logger options +// +// If failed, +// configured but initialize failed: a nil log data getter and a non-nil error will be returned. +// no getter configured: a nil log data getter with a nil error are returned +// Otherwise, a non nil log data getter is returned with nil error. +func GetLogDataGetter(loggerOptions ...Option) (getter.Interface, error) { + if len(loggerOptions) == 0 { + // If no options, directly return nil interface with error + return nil, errors.New("no options provided to create log data getter") + } + + lOptions := &options{ + values: make(map[string][]OptionItem), + } + + // Config + for _, op := range loggerOptions { + op.Apply(lOptions) + } + + // Iterate with specified order + keys := []string{} + for k := range lOptions.values { + keys = append(keys, k) + } + + // Sort + sort.Strings(keys) + + for _, k := range keys { + if HasGetter(k) { + // 1st match + d := knownLoggers[k] + theGetter, err := d.Getter(lOptions.values[k]...) + if err != nil { + return nil, err + } + + return theGetter, nil + } + } + + // No one configured + return nil, nil +} + +// Init the loggers and sweepers +func Init(ctx context.Context) error { + // For loggers + options := []Option{} + // For sweepers + sOptions := []Option{} + + for _, lc := range config.DefaultConfig.LoggerConfigs { + options = append(options, BackendOption(lc.Name, lc.Level, lc.Settings)) + if lc.Sweeper != nil { + sOptions = append(sOptions, SweeperOption(lc.Name, lc.Sweeper.Duration, lc.Sweeper.Settings)) + } + } + + // Get loggers for job service + lg, err := GetLogger(options...) + if err != nil { + return err + } + jobServiceLogger = lg + + jOptions := []Option{} + // Append configured sweepers in job loggers if existing + for _, lc := range config.DefaultConfig.JobLoggerConfigs { + jOptions = append(jOptions, BackendOption(lc.Name, lc.Level, lc.Settings)) + if lc.Sweeper != nil { + sOptions = append(sOptions, SweeperOption(lc.Name, lc.Sweeper.Duration, lc.Sweeper.Settings)) + } + } + + // Get log data getter with the same options of job loggers + g, err := GetLogDataGetter(jOptions...) + if err != nil { + return err + } + if g != nil { + logDataGetter = g + } + + // If sweepers configured + if len(sOptions) > 0 { + // Get the sweeper controller + sweeper, err := GetSweeper(ctx, sOptions...) + if err != nil { + return fmt.Errorf("create logger sweeper error: %s", err) + } + // Start sweep loop + _, err = sweeper.Sweep() + if err != nil { + return fmt.Errorf("start logger sweeper error: %s", err) + } + } + + return nil +} diff --git a/src/jobservice/logger/entry.go b/src/jobservice/logger/entry.go new file mode 100644 index 000000000..f32e798b3 --- /dev/null +++ b/src/jobservice/logger/entry.go @@ -0,0 +1,84 @@ +package logger + +// Entry provides unique interfaces on top of multiple logger backends. +// Entry also implements @Interface. +type Entry struct { + loggers []Interface +} + +// NewEntry creates a new logger Entry +func NewEntry(loggers []Interface) *Entry { + return &Entry{ + loggers: loggers, + } +} + +// Debug ... +func (e *Entry) Debug(v ...interface{}) { + for _, l := range e.loggers { + l.Debug(v...) + } +} + +// Debugf with format +func (e *Entry) Debugf(format string, v ...interface{}) { + for _, l := range e.loggers { + l.Debugf(format, v...) + } +} + +// Info ... +func (e *Entry) Info(v ...interface{}) { + for _, l := range e.loggers { + l.Info(v...) + } +} + +// Infof with format +func (e *Entry) Infof(format string, v ...interface{}) { + for _, l := range e.loggers { + l.Infof(format, v...) + } +} + +// Warning ... +func (e *Entry) Warning(v ...interface{}) { + for _, l := range e.loggers { + l.Warning(v...) + } +} + +// Warningf with format +func (e *Entry) Warningf(format string, v ...interface{}) { + for _, l := range e.loggers { + l.Warningf(format, v...) + } +} + +// Error ... +func (e *Entry) Error(v ...interface{}) { + for _, l := range e.loggers { + l.Error(v...) + } +} + +// Errorf with format +func (e *Entry) Errorf(format string, v ...interface{}) { + for _, l := range e.loggers { + l.Errorf(format, v...) + } +} + +// Fatal error +func (e *Entry) Fatal(v ...interface{}) { + for _, l := range e.loggers { + l.Fatal(v...) + } +} + +// Fatalf error +func (e *Entry) Fatalf(format string, v ...interface{}) { + for _, l := range e.loggers { + l.Fatalf(format, v...) + } +} diff --git a/src/jobservice/logger/factory.go b/src/jobservice/logger/factory.go new file mode 100644 index 000000000..32ab3f82d --- /dev/null +++ b/src/jobservice/logger/factory.go @@ -0,0 +1,54 @@ +package logger + +import ( + "errors" + "path" + + "github.com/goharbor/harbor/src/jobservice/logger/backend" +) + +// Factory creates a new logger based on the settings. +type Factory func(options ...OptionItem) (Interface, error) + +// FileFactory is factory of file logger +func FileFactory(options ...OptionItem) (Interface, error) { + var level, baseDir, fileName string + for _, op := range options { + switch op.Field() { + case "level": + level = op.String() + case "base_dir": + baseDir = op.String() + case "filename": + fileName = op.String() + default: + + } + } + + if len(baseDir) == 0 { + return nil, errors.New("missing base dir option of the file logger") + } + + if len(fileName) == 0 { + return nil, errors.New("missing file name option of the file logger") + } + + return backend.NewFileLogger(level, path.Join(baseDir, fileName)) +} + +// StdFactory is factory of std output logger. +func StdFactory(options ...OptionItem) (Interface, error) { + var level, output string + for _, op := range options { + switch op.Field() { + case "level": + level = op.String() + case "output": + output = op.String() + default: + } + } + + return backend.NewStdOutputLogger(level, output), nil +} diff --git a/src/jobservice/logger/getter/Interface.go b/src/jobservice/logger/getter/Interface.go new file mode 100644 index 000000000..206bf4a8f --- /dev/null +++ b/src/jobservice/logger/getter/Interface.go @@ -0,0 +1,12 @@ +package getter + +// Interface defines operations of a log data getter +type Interface interface { + // Retrieve the log data of the specified log entry + // + // logID string : the id of the log entry. e.g: file name a.log for file log + // + // If succeed, log data bytes will be returned + // otherwise, a non nil error is returned + Retrieve(logID string) ([]byte, error) +} diff --git a/src/jobservice/logger/getter/file_getter.go b/src/jobservice/logger/getter/file_getter.go new file mode 100644 index 000000000..630c35194 --- /dev/null +++ b/src/jobservice/logger/getter/file_getter.go @@ -0,0 +1,37 @@ +package getter + +import ( + "errors" + "fmt" + "io/ioutil" + "path" + + "github.com/goharbor/harbor/src/jobservice/errs" + + "github.com/goharbor/harbor/src/jobservice/utils" +) + +// FileGetter is responsible for retrieving file log data +type FileGetter struct { + baseDir string +} + +// NewFileGetter is constructor of FileGetter +func NewFileGetter(baseDir string) *FileGetter { + return &FileGetter{baseDir} +} + +// Retrieve implements @Interface.Retrieve +func (fg *FileGetter) Retrieve(logID string) ([]byte, error) { + if len(logID) == 0 { + return nil, errors.New("empty log identify") + } + + fPath := path.Join(fg.baseDir, fmt.Sprintf("%s.log", logID)) + + if !utils.FileExists(fPath) { + return nil, errs.NoObjectFoundError(logID) + } + + return ioutil.ReadFile(fPath) +} diff --git a/src/jobservice/logger/getter_factory.go b/src/jobservice/logger/getter_factory.go new file mode 100644 index 000000000..fdd04071a --- /dev/null +++ b/src/jobservice/logger/getter_factory.go @@ -0,0 +1,27 @@ +package logger + +import ( + "errors" + + "github.com/goharbor/harbor/src/jobservice/logger/getter" +) + +// GetterFactory is responsible for creating a log data getter based on the options +type GetterFactory func(options ...OptionItem) (getter.Interface, error) + +// FileGetterFactory creates a getter for the "FILE" logger +func FileGetterFactory(options ...OptionItem) (getter.Interface, error) { + var baseDir string + for _, op := range options { + if op.Field() == "base_dir" { + baseDir = op.String() + break + } + } + + if len(baseDir) == 0 { + return nil, errors.New("missing required option 'base_dir'") + } + + return getter.NewFileGetter(baseDir), nil +} diff --git a/src/jobservice/logger/known_loggers.go b/src/jobservice/logger/known_loggers.go new file mode 100644 index 000000000..fb69bcdcf --- /dev/null +++ b/src/jobservice/logger/known_loggers.go @@ -0,0 +1,81 @@ +package logger + +import "strings" + +const ( + // LoggerNameFile is unique name of the file logger. + LoggerNameFile = "FILE" + // LoggerNameStdOutput is the unique name of the std logger. + LoggerNameStdOutput = "STD_OUTPUT" +) + +// Declaration is used to declare a supported logger. +// Use this declaration to indicate what logger and sweeper will be provided. +type Declaration struct { + Logger Factory + Sweeper SweeperFactory + Getter GetterFactory + // Indicate if the logger is a singleton logger + Singleton bool +} + +// knownLoggers is a static logger registry. +// All the implemented loggers (w/ sweeper) should be registered +// with an unique name in this registry. Then they can be used to +// log info. +var knownLoggers = map[string]*Declaration{ + // File logger + LoggerNameFile: {FileFactory, FileSweeperFactory, FileGetterFactory, false}, + // STD output(both stdout and stderr) logger + LoggerNameStdOutput: {StdFactory, nil, nil, true}, +} + +// IsKnownLogger checks if the logger is supported with name. +func IsKnownLogger(name string) bool { + _, ok := knownLoggers[name] + + return ok +} + +// HasSweeper checks if the logger with the name provides a sweeper. +func HasSweeper(name string) bool { + d, ok := knownLoggers[name] + + return ok && d.Sweeper != nil +} + +// HasGetter checks if the logger with the name provides a log data getter. +func HasGetter(name string) bool { + d, ok := knownLoggers[name] + + return ok && d.Getter != nil +} + +// KnownLoggers return the declaration by the name +func KnownLoggers(name string) *Declaration { + return knownLoggers[name] +} + +// All known levels which are supported. +var debugLevels = []string{ + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "FATAL", +} + +// IsKnownLevel is used to check if the logger level is supported. +func IsKnownLevel(level string) bool { + if len(level) == 0 { + return false + } + + for _, lvl := range debugLevels { + if lvl == strings.ToUpper(level) { + return true + } + } + + return false +} diff --git a/src/jobservice/logger/log_data_handler.go b/src/jobservice/logger/log_data_handler.go new file mode 100644 index 000000000..cc772bf21 --- /dev/null +++ b/src/jobservice/logger/log_data_handler.go @@ -0,0 +1,23 @@ +package logger + +import ( + "errors" + + "github.com/goharbor/harbor/src/jobservice/logger/getter" +) + +var logDataGetter getter.Interface + +// Retrieve is wrapper func for getter.Retrieve +func Retrieve(logID string) ([]byte, error) { + if logDataGetter == nil { + return nil, errors.New("no log data getter is configured") + } + + return logDataGetter.Retrieve(logID) +} + +// HasLogGetterConfigured checks if a log data getter is there for using +func HasLogGetterConfigured() bool { + return logDataGetter != nil +} diff --git a/src/jobservice/logger/options.go b/src/jobservice/logger/options.go new file mode 100644 index 000000000..ad00d1345 --- /dev/null +++ b/src/jobservice/logger/options.go @@ -0,0 +1,99 @@ +package logger + +// options keep settings for loggers/sweepers +// Indexed by the logger unique name +type options struct { + values map[string][]OptionItem +} + +// Option represents settings of the logger +type Option struct { + // Apply logger option + Apply func(op *options) +} + +// BackendOption creates option for the specified backend. +func BackendOption(name string, level string, settings map[string]interface{}) Option { + return Option{func(op *options) { + vals := make([]OptionItem, 0) + vals = append(vals, OptionItem{"level", level}) + + // Append extra settings if existing + if len(settings) > 0 { + for k, v := range settings { + vals = append(vals, OptionItem{k, v}) + } + } + + // Append with overriding way + op.values[name] = vals + }} +} + +// SweeperOption creates option for the sweeper. +func SweeperOption(name string, duration int, settings map[string]interface{}) Option { + return Option{func(op *options) { + vals := make([]OptionItem, 0) + vals = append(vals, OptionItem{"duration", duration}) + + // Append settings if existing + if len(settings) > 0 { + for k, v := range settings { + vals = append(vals, OptionItem{k, v}) + } + } + + // Append with overriding way + op.values[name] = vals + }} +} + +// GetterOption creates option for the getter. +func GetterOption(name string, settings map[string]interface{}) Option { + return Option{func(op *options) { + vals := make([]OptionItem, 0) + // Append settings if existing + if len(settings) > 0 { + for k, v := range settings { + vals = append(vals, OptionItem{k, v}) + } + } + + // Append with overriding way + op.values[name] = vals + }} +} + +// OptionItem is a simple wrapper of property and value +type OptionItem struct { + field string + val interface{} +} + +// Field returns name of the option +func (o *OptionItem) Field() string { + return o.field +} + +// Int returns the integer value of option +func (o *OptionItem) Int() int { + if o.val == nil { + return 0 + } + + return o.val.(int) +} + +// String returns the string value of option +func (o *OptionItem) String() string { + if o.val == nil { + return "" + } + + return o.val.(string) +} + +// Raw returns the raw value +func (o *OptionItem) Raw() interface{} { + return o.val +} diff --git a/src/jobservice/logger/service_logger.go b/src/jobservice/logger/service.go similarity index 69% rename from src/jobservice/logger/service_logger.go rename to src/jobservice/logger/service.go index 9e1740359..e39e3e336 100644 --- a/src/jobservice/logger/service_logger.go +++ b/src/jobservice/logger/service.go @@ -18,18 +18,13 @@ import ( "log" ) -// sLogger is used to log for workerpool itself -var sLogger Interface - -// SetLogger sets the logger implementation -func SetLogger(logger Interface) { - sLogger = logger -} +// jobServiceLogger is used to log for job service itself +var jobServiceLogger Interface // Debug ... func Debug(v ...interface{}) { - if sLogger != nil { - sLogger.Debug(v...) + if jobServiceLogger != nil { + jobServiceLogger.Debug(v...) return } @@ -38,8 +33,8 @@ func Debug(v ...interface{}) { // Debugf for debuging with format func Debugf(format string, v ...interface{}) { - if sLogger != nil { - sLogger.Debugf(format, v...) + if jobServiceLogger != nil { + jobServiceLogger.Debugf(format, v...) return } @@ -48,8 +43,8 @@ func Debugf(format string, v ...interface{}) { // Info ... func Info(v ...interface{}) { - if sLogger != nil { - sLogger.Info(v...) + if jobServiceLogger != nil { + jobServiceLogger.Info(v...) return } @@ -58,8 +53,8 @@ func Info(v ...interface{}) { // Infof for logging info with format func Infof(format string, v ...interface{}) { - if sLogger != nil { - sLogger.Infof(format, v...) + if jobServiceLogger != nil { + jobServiceLogger.Infof(format, v...) return } @@ -68,8 +63,8 @@ func Infof(format string, v ...interface{}) { // Warning ... func Warning(v ...interface{}) { - if sLogger != nil { - sLogger.Warning(v...) + if jobServiceLogger != nil { + jobServiceLogger.Warning(v...) return } @@ -78,8 +73,8 @@ func Warning(v ...interface{}) { // Warningf for warning with format func Warningf(format string, v ...interface{}) { - if sLogger != nil { - sLogger.Warningf(format, v...) + if jobServiceLogger != nil { + jobServiceLogger.Warningf(format, v...) return } @@ -88,8 +83,8 @@ func Warningf(format string, v ...interface{}) { // Error for logging error func Error(v ...interface{}) { - if sLogger != nil { - sLogger.Error(v...) + if jobServiceLogger != nil { + jobServiceLogger.Error(v...) return } @@ -98,8 +93,8 @@ func Error(v ...interface{}) { // Errorf for logging error with format func Errorf(format string, v ...interface{}) { - if sLogger != nil { - sLogger.Errorf(format, v...) + if jobServiceLogger != nil { + jobServiceLogger.Errorf(format, v...) return } @@ -108,8 +103,8 @@ func Errorf(format string, v ...interface{}) { // Fatal ... func Fatal(v ...interface{}) { - if sLogger != nil { - sLogger.Fatal(v...) + if jobServiceLogger != nil { + jobServiceLogger.Fatal(v...) return } @@ -118,8 +113,8 @@ func Fatal(v ...interface{}) { // Fatalf for fatal error with error func Fatalf(format string, v ...interface{}) { - if sLogger != nil { - sLogger.Fatalf(format, v...) + if jobServiceLogger != nil { + jobServiceLogger.Fatalf(format, v...) return } diff --git a/src/jobservice/logger/service_logger_test.go b/src/jobservice/logger/service_logger_test.go deleted file mode 100644 index 1cbc4faf0..000000000 --- a/src/jobservice/logger/service_logger_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package logger - -import ( - "fmt" - "testing" -) - -func TestServiceLogger(t *testing.T) { - testingLogger := &fakeLogger{} - SetLogger(testingLogger) - - Debug("DEBUG") - Debugf("%s\n", "DEBUGF") - Info("INFO") - Infof("%s\n", "INFOF") - Warning("WARNING") - Warningf("%s\n", "WARNINGF") - Error("ERROR") - Errorf("%s\n", "ERRORF") - Fatal("FATAL") - Fatalf("%s\n", "FATALF") -} - -type fakeLogger struct{} - -// For debuging -func (fl *fakeLogger) Debug(v ...interface{}) { - fmt.Println(v...) -} - -// For debuging with format -func (fl *fakeLogger) Debugf(format string, v ...interface{}) { - fmt.Printf(format, v...) -} - -// For logging info -func (fl *fakeLogger) Info(v ...interface{}) { - fmt.Println(v...) -} - -// For logging info with format -func (fl *fakeLogger) Infof(format string, v ...interface{}) { - fmt.Printf(format, v...) -} - -// For warning -func (fl *fakeLogger) Warning(v ...interface{}) { - fmt.Println(v...) -} - -// For warning with format -func (fl *fakeLogger) Warningf(format string, v ...interface{}) { - fmt.Printf(format, v...) -} - -// For logging error -func (fl *fakeLogger) Error(v ...interface{}) { - fmt.Println(v...) -} - -// For logging error with format -func (fl *fakeLogger) Errorf(format string, v ...interface{}) { - fmt.Printf(format, v...) -} - -// For fatal error -func (fl *fakeLogger) Fatal(v ...interface{}) { - fmt.Println(v...) -} - -// For fatal error with error -func (fl *fakeLogger) Fatalf(format string, v ...interface{}) { - fmt.Printf(format, v...) -} diff --git a/src/jobservice/logger/sweeper.go b/src/jobservice/logger/sweeper.go deleted file mode 100644 index 3682c5307..000000000 --- a/src/jobservice/logger/sweeper.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logger - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "time" -) - -const ( - oneDay = 3600 * 24 -) - -// Sweeper takes charge of archive the outdated log files of jobs. -type Sweeper struct { - context context.Context - workDir string - period uint -} - -// NewSweeper creates new prt of Sweeper -func NewSweeper(ctx context.Context, workDir string, period uint) *Sweeper { - return &Sweeper{ctx, workDir, period} -} - -// Start to work -func (s *Sweeper) Start() { - go s.loop() - Info("Logger sweeper is started") -} - -func (s *Sweeper) loop() { - // Apply default if needed before starting - if s.period == 0 { - s.period = 1 - } - - defer func() { - Info("Logger sweeper is stopped") - }() - - // First run - go s.clear() - - ticker := time.NewTicker(time.Duration(s.period*oneDay+5) * time.Second) - defer ticker.Stop() - - for { - select { - case <-s.context.Done(): - return - case <-ticker.C: - go s.clear() - } - } -} - -func (s *Sweeper) clear() { - var ( - cleared uint - count = &cleared - ) - - Info("Start to clear the job outdated log files") - defer func() { - Infof("%d job outdated log files cleared", *count) - }() - - logFiles, err := ioutil.ReadDir(s.workDir) - if err != nil { - Errorf("Failed to get the outdated log files under '%s' with error: %s\n", s.workDir, err) - return - } - if len(logFiles) == 0 { - return - } - - for _, logFile := range logFiles { - if logFile.ModTime().Add(time.Duration(s.period*oneDay) * time.Second).Before(time.Now()) { - logFilePath := fmt.Sprintf("%s%s%s", s.workDir, string(os.PathSeparator), logFile.Name()) - if err := os.Remove(logFilePath); err == nil { - cleared++ - } else { - Warningf("Failed to remove log file '%s'\n", logFilePath) - } - } - } -} diff --git a/src/jobservice/logger/sweeper/file_sweeper.go b/src/jobservice/logger/sweeper/file_sweeper.go new file mode 100644 index 000000000..6a2229593 --- /dev/null +++ b/src/jobservice/logger/sweeper/file_sweeper.go @@ -0,0 +1,70 @@ +package sweeper + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "strings" + "time" +) + +const ( + oneDay = 24 * time.Hour +) + +// FileSweeper is used to sweep the file logs +type FileSweeper struct { + duration int + workDir string +} + +// NewFileSweeper is constructor of FileSweeper +func NewFileSweeper(workDir string, duration int) *FileSweeper { + return &FileSweeper{ + workDir: workDir, + duration: duration, + } +} + +// Sweep logs +func (fs *FileSweeper) Sweep() (int, error) { + cleared := 0 + + logFiles, err := ioutil.ReadDir(fs.workDir) + if err != nil { + return 0, fmt.Errorf("getting outdated log files under '%s' failed with error: %s", fs.workDir, err) + } + + // Nothing to sweep + if len(logFiles) == 0 { + return 0, nil + } + + // Start to sweep log files + // Record all errors + errs := make([]string, 0) + for _, logFile := range logFiles { + if logFile.ModTime().Add(time.Duration(fs.duration) * oneDay).Before(time.Now()) { + logFilePath := path.Join(fs.workDir, logFile.Name()) + if err := os.Remove(logFilePath); err != nil { + errs = append(errs, fmt.Sprintf("remove log file '%s' error: %s", logFilePath, err)) + continue // go on for next one + } + + cleared++ + } + } + + if len(errs) > 0 { + err = fmt.Errorf("%s", strings.Join(errs, "\n")) + } + + // Return error with high priority + return cleared, err +} + +// Duration for sweeping +func (fs *FileSweeper) Duration() int { + return fs.duration +} diff --git a/src/jobservice/logger/sweeper/interface.go b/src/jobservice/logger/sweeper/interface.go new file mode 100644 index 000000000..a57ef6099 --- /dev/null +++ b/src/jobservice/logger/sweeper/interface.go @@ -0,0 +1,13 @@ +package sweeper + +// Interface defines the operations a sweeper should have +type Interface interface { + // Sweep the outdated log entries if necessary + // + // If failed, an non-nil error will return + // If succeeded, count of sweepped log entries is returned + Sweep() (int, error) + + // Return the sweeping duration with day unit. + Duration() int +} diff --git a/src/jobservice/logger/sweeper_controller.go b/src/jobservice/logger/sweeper_controller.go new file mode 100644 index 000000000..53d1a7769 --- /dev/null +++ b/src/jobservice/logger/sweeper_controller.go @@ -0,0 +1,98 @@ +package logger + +import ( + "context" + "fmt" + "reflect" + "time" + + "github.com/goharbor/harbor/src/jobservice/logger/sweeper" +) + +const ( + oneDay = 24 * time.Hour +) + +// SweeperController is an unified sweeper entry and built on top of multiple sweepers. +// It's responsible for starting the configured sweepers. +type SweeperController struct { + context context.Context + sweepers []sweeper.Interface + errChan chan error +} + +// NewSweeperController is constructor of controller. +func NewSweeperController(ctx context.Context, sweepers []sweeper.Interface) *SweeperController { + return &SweeperController{ + context: ctx, + sweepers: sweepers, + errChan: make(chan error, 1), + } +} + +// Sweep logs +func (c *SweeperController) Sweep() (int, error) { + // Start to process errors + go func() { + for { + select { + case err := <-c.errChan: + Error(err) + case <-c.context.Done(): + return + } + } + }() + + for _, s := range c.sweepers { + go func(sw sweeper.Interface) { + c.startSweeper(sw) + }(s) + } + + return 0, nil +} + +// Duration = -1 for controller +func (c *SweeperController) Duration() int { + return -1 +} + +func (c *SweeperController) startSweeper(s sweeper.Interface) { + d := s.Duration() + if d <= 0 { + d = 1 + } + + // Use the type name as a simple ID + sid := reflect.TypeOf(s).String() + defer Infof("sweeper %s exit", sid) + + // First run + go c.doSweeping(sid, s) + + // Loop + ticker := time.NewTicker(time.Duration(d) * oneDay) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + go c.doSweeping(sid, s) + case <-c.context.Done(): + return + } + } +} + +func (c *SweeperController) doSweeping(sid string, s sweeper.Interface) { + Debugf("Sweeper %s is under working", sid) + + count, err := s.Sweep() + if err != nil { + c.errChan <- fmt.Errorf("sweep logs error in %s at %d: %s", sid, time.Now().Unix(), err) + return + } + + Infof("%d outdated log entries are sweepped by sweeper %s", count, sid) +} diff --git a/src/jobservice/logger/sweeper_factory.go b/src/jobservice/logger/sweeper_factory.go new file mode 100644 index 000000000..5c45c0f31 --- /dev/null +++ b/src/jobservice/logger/sweeper_factory.go @@ -0,0 +1,32 @@ +package logger + +import ( + "errors" + + "github.com/goharbor/harbor/src/jobservice/logger/sweeper" +) + +// SweeperFactory is responsible for creating a sweeper.Interface based on the settings +type SweeperFactory func(options ...OptionItem) (sweeper.Interface, error) + +// FileSweeperFactory creates file sweeper. +func FileSweeperFactory(options ...OptionItem) (sweeper.Interface, error) { + var workDir, duration = "", 1 + for _, op := range options { + switch op.Field() { + case "work_dir": + workDir = op.String() + case "duration": + if op.Int() > 0 { + duration = op.Int() + } + default: + } + } + + if len(workDir) == 0 { + return nil, errors.New("missing required option 'work_dir'") + } + + return sweeper.NewFileSweeper(workDir, duration), nil +} diff --git a/src/jobservice/logger/sweeper_test.go b/src/jobservice/logger/sweeper_test.go deleted file mode 100644 index dd75dd976..000000000 --- a/src/jobservice/logger/sweeper_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package logger - -import ( - "context" - "fmt" - "os" - "testing" - "time" -) - -func TestSweeper(t *testing.T) { - workDir := "/tmp/sweeper_logs" - - if err := os.MkdirAll(workDir, 0755); err != nil { - t.Fatal(err) - } - _, err := os.Create(fmt.Sprintf("%s/sweeper_test.log", workDir)) - if err != nil { - t.Fatal(err) - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sweeper := NewSweeper(ctx, workDir, 1) - sweeper.Start() - <-time.After(100 * time.Millisecond) - - if err := os.Remove(fmt.Sprintf("%s/sweeper_test.log", workDir)); err != nil { - t.Fatal(err) - } - if err := os.Remove(workDir); err != nil { - t.Fatal(err) - } -} diff --git a/src/jobservice/main.go b/src/jobservice/main.go index 650c837bb..d4e6cf1f5 100644 --- a/src/jobservice/main.go +++ b/src/jobservice/main.go @@ -15,14 +15,11 @@ package main import ( - "errors" + "context" "flag" + "fmt" - "github.com/goharbor/harbor/src/adminserver/client" "github.com/goharbor/harbor/src/jobservice/config" - "github.com/goharbor/harbor/src/jobservice/env" - "github.com/goharbor/harbor/src/jobservice/job/impl" - ilogger "github.com/goharbor/harbor/src/jobservice/job/impl/logger" "github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/jobservice/runtime" "github.com/goharbor/harbor/src/jobservice/utils" @@ -36,16 +33,25 @@ func main() { // Missing config file if configPath == nil || utils.IsEmptyStr(*configPath) { flag.Usage() - logger.Fatal("Config file should be specified") + panic("no config file is specified") } // Load configurations if err := config.DefaultConfig.Load(*configPath, true); err != nil { - logger.Fatalf("Failed to load configurations with error: %s\n", err) + panic(fmt.Sprintf("load configurations error: %s\n", err)) + } + + // Create the root context + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Initialize logger + if err := logger.Init(ctx); err != nil { + panic(err) } // Set job context initializer - runtime.JobService.SetJobContextInitializer(func(ctx *env.Context) (env.JobContext, error) { + /*runtime.JobService.SetJobContextInitializer(func(ctx *env.Context) (env.JobContext, error) { secret := config.GetAuthSecret() if utils.IsEmptyStr(secret) { return nil, errors.New("empty auth secret") @@ -59,12 +65,8 @@ func main() { } return jobCtx, nil - }) - - // New logger for job service - sLogger := ilogger.NewServiceLogger(config.GetLogLevel()) - logger.SetLogger(sLogger) + })*/ // Start - runtime.JobService.LoadAndRun() + runtime.JobService.LoadAndRun(ctx, cancel) } diff --git a/src/jobservice/runtime/bootstrap.go b/src/jobservice/runtime/bootstrap.go index 1f511e9f9..d8937b924 100644 --- a/src/jobservice/runtime/bootstrap.go +++ b/src/jobservice/runtime/bootstrap.go @@ -64,11 +64,7 @@ func (bs *Bootstrap) SetJobContextInitializer(initializer env.JobContextInitiali // LoadAndRun will load configurations, initialize components and then start the related process to serve requests. // Return error if meet any problems. -func (bs *Bootstrap) LoadAndRun() { - // Create the root context - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - +func (bs *Bootstrap) LoadAndRun(ctx context.Context, cancel context.CancelFunc) { rootContext := &env.Context{ SystemContext: ctx, WG: &sync.WaitGroup{}, @@ -110,10 +106,6 @@ func (bs *Bootstrap) LoadAndRun() { apiServer := bs.loadAndRunAPIServer(rootContext, config.DefaultConfig, ctl) logger.Infof("Server is started at %s:%d with %s", "", config.DefaultConfig.Port, config.DefaultConfig.Protocol) - // Start outdated log files sweeper - logSweeper := logger.NewSweeper(ctx, config.GetLogBasePath(), config.GetLogArchivePeriod()) - logSweeper.Start() - // To indicate if any errors occurred var err error // Block here