Build logger framework to support configurable loggers/sweepers/getters

Signed-off-by: Steven Zou <szou@vmware.com>

src/common/utils/log/logger.go

@@ -60,7 +60,7 @@ func New(out io.Writer, fmtter Formatter, lvl Level) *Logger {
 		out:       out,
 		fmtter:    fmtter,
 		lvl:       lvl,
-		callDepth: 3,
+		callDepth: 6,
 	}
 }
 

src/jobservice/config.yml

@@ -22,11 +22,23 @@ worker_pool:
     redis_url: "localhost:6379"
     namespace: "harbor_job_service"
 
-#Logger for job
+#Loggers for the running job
-logger:
+job_loggers:
-  path: "~/tmp/job_logs"
+  - name: "STD_OUTPUT" # logger backend name, only support "FILE" and "STD_OUTPUT"
+    level: "DEBUG" # INFO/DEBUG/WARNING/ERROR/FATAL
+  - name: "FILE"
+    level: "DEBUG"
+    settings: # Customized settings of logger
+      base_dir: "/Users/szou/tmp/job_logs"
+    sweeper:
+      duration: 1 #days
+      settings: # Customized settings of sweeper
+        work_dir: "/Users/szou/tmp/job_logs"
+
+#Loggers for the job service
+loggers:
+  - name: "STD_OUTPUT" # Same with above
     level: "DEBUG"
-  archive_period: 1 #days
 
 #Admin server endpoint
 admin_server: "http://adminserver:9010/"

src/jobservice/config/config.go

@@ -36,9 +36,6 @@ const (
 	jobServiceWorkers            = "JOB_SERVICE_POOL_WORKERS"
 	jobServiceRedisURL           = "JOB_SERVICE_POOL_REDIS_URL"
 	jobServiceRedisNamespace     = "JOB_SERVICE_POOL_REDIS_NAMESPACE"
-	jobServiceLoggerBasePath      = "JOB_SERVICE_LOGGER_BASE_PATH"
-	jobServiceLoggerLevel         = "JOB_SERVICE_LOGGER_LEVEL"
-	jobServiceLoggerArchivePeriod = "JOB_SERVICE_LOGGER_ARCHIVE_PERIOD"
 	jobServiceCoreServerEndpoint = "CORE_URL"
 	jobServiceAuthSecret         = "JOBSERVICE_SECRET"
 
@@ -76,8 +73,11 @@ type Configuration struct {
 	// Configurations of worker pool
 	PoolConfig *PoolConfig `yaml:"worker_pool,omitempty"`
 
+	// Job logger configurations
+	JobLoggerConfigs []*LoggerConfig `yaml:"job_loggers,omitempty"`
+
 	// Logger configurations
-	LoggerConfig *LoggerConfig `yaml:"logger,omitempty"`
+	LoggerConfigs []*LoggerConfig `yaml:"loggers,omitempty"`
 }
 
 // HTTPSConfig keeps additional configurations when using https protocol
@@ -100,11 +100,21 @@ type PoolConfig struct {
 	RedisPoolCfg *RedisPoolConfig `yaml:"redis_pool,omitempty"`
 }
 
-// LoggerConfig keeps logger configurations.
+// CustomizedSettings keeps the customized settings of logger
+type CustomizedSettings map[string]interface{}
+
+// LogSweeperConfig keeps settings of log sweeper
+type LogSweeperConfig struct {
+	Duration int                `yaml:"duration"`
+	Settings CustomizedSettings `yaml:"settings"`
+}
+
+// LoggerConfig keeps logger basic configurations.
 type LoggerConfig struct {
-	BasePath      string `yaml:"path"`
+	Name     string             `yaml:"name"`
-	LogLevel      string `yaml:"level"`
+	Level    string             `yaml:"level"`
-	ArchivePeriod uint   `yaml:"archive_period"`
+	Settings CustomizedSettings `yaml:"settings"`
+	Sweeper  *LogSweeperConfig  `yaml:"sweeper"`
 }
 
 // Load the configuration options from the specified yaml file.
@@ -151,33 +161,6 @@ func (c *Configuration) Load(yamlFilePath string, detectEnv bool) error {
 	return c.validate()
 }
 
-// GetLogBasePath returns the log base path config
-func GetLogBasePath() string {
-	if DefaultConfig.LoggerConfig != nil {
-		return DefaultConfig.LoggerConfig.BasePath
-	}
-
-	return ""
-}
-
-// GetLogLevel returns the log level
-func GetLogLevel() string {
-	if DefaultConfig.LoggerConfig != nil {
-		return DefaultConfig.LoggerConfig.LogLevel
-	}
-
-	return ""
-}
-
-// GetLogArchivePeriod returns the archive period
-func GetLogArchivePeriod() uint {
-	if DefaultConfig.LoggerConfig != nil {
-		return DefaultConfig.LoggerConfig.ArchivePeriod
-	}
-
-	return 1 // return default
-}
-
 // GetAuthSecret get the auth secret from the env
 func GetAuthSecret() string {
 	return utils.ReadEnv(jobServiceAuthSecret)
@@ -268,31 +251,6 @@ func (c *Configuration) loadEnvs() {
 		}
 	}
 
-	// logger
-	loggerPath := utils.ReadEnv(jobServiceLoggerBasePath)
-	if !utils.IsEmptyStr(loggerPath) {
-		if c.LoggerConfig == nil {
-			c.LoggerConfig = &LoggerConfig{}
-		}
-		c.LoggerConfig.BasePath = loggerPath
-	}
-	loggerLevel := utils.ReadEnv(jobServiceLoggerLevel)
-	if !utils.IsEmptyStr(loggerLevel) {
-		if c.LoggerConfig == nil {
-			c.LoggerConfig = &LoggerConfig{}
-		}
-		c.LoggerConfig.LogLevel = loggerLevel
-	}
-	archivePeriod := utils.ReadEnv(jobServiceLoggerArchivePeriod)
-	if !utils.IsEmptyStr(archivePeriod) {
-		if period, err := strconv.Atoi(archivePeriod); err == nil {
-			if c.LoggerConfig == nil {
-				c.LoggerConfig = &LoggerConfig{}
-			}
-			c.LoggerConfig.ArchivePeriod = uint(period)
-		}
-	}
-
 	// admin server
 	if coreServer := utils.ReadEnv(jobServiceCoreServerEndpoint); !utils.IsEmptyStr(coreServer) {
 		c.AdminServer = coreServer
@@ -357,21 +315,14 @@ func (c *Configuration) validate() error {
 		}
 	}
 
-	if c.LoggerConfig == nil {
+	// Job service loggers
-		return errors.New("missing logger config")
+	if len(c.LoggerConfigs) == 0 {
+		return errors.New("missing logger config of job service")
 	}
 
-	if !utils.DirExists(c.LoggerConfig.BasePath) {
+	// Job loggers
-		return errors.New("logger path should be an existing dir")
+	if len(c.JobLoggerConfigs) == 0 {
-	}
+		return errors.New("missing logger config of job")
-
-	validLevels := "DEBUG,INFO,WARNING,ERROR,FATAL"
-	if !strings.Contains(validLevels, c.LoggerConfig.LogLevel) {
-		return fmt.Errorf("logger level can only be one of: %s", validLevels)
-	}
-
-	if c.LoggerConfig.ArchivePeriod == 0 {
-		return fmt.Errorf("logger archive period should be greater than 0")
 	}
 
 	if _, err := url.Parse(c.AdminServer); err != nil {

src/jobservice/config/config_test.go

@@ -26,24 +26,13 @@ func TestConfigLoadingFailed(t *testing.T) {
 }
 
 func TestConfigLoadingSucceed(t *testing.T) {
-	if err := CreateLogDir(); err != nil {
-		t.Fatal(err)
-	}
-
 	cfg := &Configuration{}
 	if err := cfg.Load("../config_test.yml", false); err != nil {
 		t.Fatalf("Load config from yaml file, expect nil error but got error '%s'\n", err)
 	}
-
-	if err := RemoveLogDir(); err != nil {
-		t.Fatal(err)
-	}
 }
 
 func TestConfigLoadingWithEnv(t *testing.T) {
-	if err := CreateLogDir(); err != nil {
-		t.Error(err)
-	}
 	setENV()
 
 	cfg := &Configuration{}
@@ -52,68 +41,74 @@ func TestConfigLoadingWithEnv(t *testing.T) {
 	}
 
 	if cfg.Protocol != "https" {
-		t.Fatalf("expect protocol 'https', but got '%s'\n", cfg.Protocol)
+		t.Errorf("expect protocol 'https', but got '%s'\n", cfg.Protocol)
 	}
 	if cfg.Port != 8989 {
-		t.Fatalf("expect port 8989 but got '%d'\n", cfg.Port)
+		t.Errorf("expect port 8989 but got '%d'\n", cfg.Port)
 	}
 	if cfg.PoolConfig.WorkerCount != 8 {
-		t.Fatalf("expect workcount 8 but go '%d'\n", cfg.PoolConfig.WorkerCount)
+		t.Errorf("expect workcount 8 but go '%d'\n", cfg.PoolConfig.WorkerCount)
 	}
 	if cfg.PoolConfig.RedisPoolCfg.RedisURL != "redis://arbitrary_username:password@8.8.8.8:6379/0" {
-		t.Fatalf("expect redis URL 'localhost' but got '%s'\n", cfg.PoolConfig.RedisPoolCfg.RedisURL)
+		t.Errorf("expect redis URL 'localhost' but got '%s'\n", cfg.PoolConfig.RedisPoolCfg.RedisURL)
 	}
 	if cfg.PoolConfig.RedisPoolCfg.Namespace != "ut_namespace" {
-		t.Fatalf("expect redis namespace 'ut_namespace' but got '%s'\n", cfg.PoolConfig.RedisPoolCfg.Namespace)
+		t.Errorf("expect redis namespace 'ut_namespace' but got '%s'\n", cfg.PoolConfig.RedisPoolCfg.Namespace)
 	}
-	if cfg.LoggerConfig.BasePath != "/tmp" {
+	if GetAuthSecret() != "js_secret" {
-		t.Fatalf("expect log base path '/tmp' but got '%s'\n", cfg.LoggerConfig.BasePath)
+		t.Errorf("expect auth secret 'js_secret' but got '%s'", GetAuthSecret())
 	}
-	if cfg.LoggerConfig.LogLevel != "DEBUG" {
+	if GetUIAuthSecret() != "core_secret" {
-		t.Fatalf("expect log level 'DEBUG' but got '%s'\n", cfg.LoggerConfig.LogLevel)
+		t.Errorf("expect auth secret 'core_secret' but got '%s'", GetUIAuthSecret())
-	}
-	if cfg.LoggerConfig.ArchivePeriod != 5 {
-		t.Fatalf("expect log archive period 5 but got '%d'\n", cfg.LoggerConfig.ArchivePeriod)
 	}
 
 	unsetENV()
-	if err := RemoveLogDir(); err != nil {
-		t.Fatal(err)
-	}
 }
 
 func TestDefaultConfig(t *testing.T) {
-	if err := CreateLogDir(); err != nil {
-		t.Fatal(err)
-	}
-
 	if err := DefaultConfig.Load("../config_test.yml", true); err != nil {
 		t.Fatalf("Load config from yaml file, expect nil error but got error '%s'\n", err)
 	}
 
 	if endpoint := GetAdminServerEndpoint(); endpoint != "http://127.0.0.1:8888" {
-		t.Fatalf("expect default admin server endpoint 'http://127.0.0.1:8888' but got '%s'\n", endpoint)
+		t.Errorf("expect default admin server endpoint 'http://127.0.0.1:8888' but got '%s'\n", endpoint)
-	}
-
-	if basePath := GetLogBasePath(); basePath != "/tmp/job_logs" {
-		t.Fatalf("expect default logger base path '/tmp/job_logs' but got '%s'\n", basePath)
-	}
-
-	if lvl := GetLogLevel(); lvl != "INFO" {
-		t.Fatalf("expect default logger level 'INFO' but got '%s'\n", lvl)
-	}
-
-	if period := GetLogArchivePeriod(); period != 1 {
-		t.Fatalf("expect default log archive period 1 but got '%d'\n", period)
 	}
 
 	redisURL := DefaultConfig.PoolConfig.RedisPoolCfg.RedisURL
-	if redisURL != "redis://redis:6379" {
+	if redisURL != "redis://localhost:6379" {
-		t.Fatalf("expect redisURL '%s' but got '%s'\n", "redis://redis:6379", redisURL)
+		t.Errorf("expect redisURL '%s' but got '%s'\n", "redis://localhost:6379", redisURL)
 	}
 
-	if err := RemoveLogDir(); err != nil {
+	if len(DefaultConfig.JobLoggerConfigs) == 0 {
-		t.Fatal(err)
+		t.Errorf("expect 2 job loggers configured but got %d", len(DefaultConfig.JobLoggerConfigs))
+	}
+
+	if len(DefaultConfig.LoggerConfigs) == 0 {
+		t.Errorf("expect 1 loggers configured but got %d", len(DefaultConfig.LoggerConfigs))
+	}
+
+	// Only verify the complicated one
+	theLogger := DefaultConfig.JobLoggerConfigs[1]
+	if theLogger.Name != "FILE" {
+		t.Fatalf("expect FILE logger but got %s", theLogger.Name)
+	}
+	if theLogger.Level != "INFO" {
+		t.Errorf("expect INFO log level of FILE logger but got %s", theLogger.Level)
+	}
+	if len(theLogger.Settings) == 0 {
+		t.Errorf("expect extra settings but got nothing")
+	}
+	if theLogger.Settings["base_dir"] != "/tmp/job_logs" {
+		t.Errorf("expect extra setting base_dir to be '/tmp/job_logs' but got %s", theLogger.Settings["base_dir"])
+	}
+	if theLogger.Sweeper == nil {
+		t.Fatalf("expect non nil sweeper of FILE logger but got nil")
+	}
+	if theLogger.Sweeper.Duration != 5 {
+		t.Errorf("expect sweep duration to be 5 but got %d", theLogger.Sweeper.Duration)
+	}
+	if theLogger.Sweeper.Settings["work_dir"] != "/tmp/job_logs" {
+		t.Errorf("expect work dir of sweeper of FILE logger to be '/tmp/job_logs' but got %s", theLogger.Sweeper.Settings["work_dir"])
 	}
 }
 
@@ -126,9 +121,8 @@ func setENV() {
 	os.Setenv("JOB_SERVICE_POOL_WORKERS", "8")
 	os.Setenv("JOB_SERVICE_POOL_REDIS_URL", "8.8.8.8:6379,100,password,0")
 	os.Setenv("JOB_SERVICE_POOL_REDIS_NAMESPACE", "ut_namespace")
-	os.Setenv("JOB_SERVICE_LOGGER_BASE_PATH", "/tmp")
+	os.Setenv("JOBSERVICE_SECRET", "js_secret")
-	os.Setenv("JOB_SERVICE_LOGGER_LEVEL", "DEBUG")
+	os.Setenv("CORE_SECRET", "core_secret")
-	os.Setenv("JOB_SERVICE_LOGGER_ARCHIVE_PERIOD", "5")
 }
 
 func unsetENV() {
@@ -140,15 +134,6 @@ func unsetENV() {
 	os.Unsetenv("JOB_SERVICE_POOL_WORKERS")
 	os.Unsetenv("JOB_SERVICE_POOL_REDIS_URL")
 	os.Unsetenv("JOB_SERVICE_POOL_REDIS_NAMESPACE")
-	os.Unsetenv("JOB_SERVICE_LOGGER_BASE_PATH")
+	os.Unsetenv("JOBSERVICE_SECRET")
-	os.Unsetenv("JOB_SERVICE_LOGGER_LEVEL")
+	os.Unsetenv("CORE_SECRET")
-	os.Unsetenv("JOB_SERVICE_LOGGER_ARCHIVE_PERIOD")
-}
-
-func CreateLogDir() error {
-	return os.MkdirAll("/tmp/job_logs", 0755)
-}
-
-func RemoveLogDir() error {
-	return os.Remove("/tmp/job_logs")
 }

src/jobservice/config_test.yml

@@ -8,7 +8,7 @@ https_config:
   key: "../server.key"
 
 #Server listening port
-port: 9443
+port: 9444
 
 #Worker pool
 worker_pool:
@@ -19,14 +19,26 @@ worker_pool:
   redis_pool:
     #redis://[arbitrary_username:password@]ipaddress:port/database_index
     #or ipaddress:port[,weight,password,database_index]
-    redis_url: "redis:6379"
+    redis_url: "localhost:6379"
     namespace: "harbor_job_service"
 
-#Logger for job
+#Loggers for the running job
-logger:
+job_loggers:
-  path: "/tmp/job_logs"
+  - name: "STD_OUTPUT" # logger backend name, only support "FILE" and "STD_OUTPUT"
+    level: "DEBUG" # INFO/DEBUG/WARNING/ERROR/FATAL
+  - name: "FILE"
     level: "INFO"
-  archive_period: 1 #days
+    settings: # Customized settings of logger
+      base_dir: "/tmp/job_logs"
+    sweeper:
+      duration: 5 #days
+      settings: # Customized settings of sweeper
+        work_dir: "/tmp/job_logs"
+
+#Loggers for the job service
+loggers:
+  - name: "STD_OUTPUT" # Same with above
+    level: "DEBUG"
 
 #Admin server endpoint
 admin_server: "http://127.0.0.1:8888"

src/jobservice/core/controller.go

@@ -17,10 +17,9 @@ package core
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
 
-	"github.com/goharbor/harbor/src/jobservice/config"
+	"github.com/goharbor/harbor/src/jobservice/logger"
-	"github.com/goharbor/harbor/src/jobservice/errs"
+
 	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/jobservice/models"
 	"github.com/goharbor/harbor/src/jobservice/pool"
@@ -141,12 +140,7 @@ func (c *Controller) GetJobLogData(jobID string) ([]byte, error) {
 		return nil, errors.New("empty job ID")
 	}
 
-	logPath := fmt.Sprintf("%s/%s.log", config.GetLogBasePath(), jobID)
+	logData, err := logger.Retrieve(jobID)
-	if !utils.FileExists(logPath) {
-		return nil, errs.NoObjectFoundError(fmt.Sprintf("%s.log", jobID))
-	}
-
-	logData, err := ioutil.ReadFile(logPath)
 	if err != nil {
 		return nil, err
 	}

src/jobservice/core/controller_test.go

@@ -17,8 +17,6 @@ import (
 	"errors"
 	"testing"
 
-	"github.com/goharbor/harbor/src/jobservice/errs"
-
 	"github.com/goharbor/harbor/src/jobservice/env"
 	"github.com/goharbor/harbor/src/jobservice/models"
 )
@@ -141,11 +139,7 @@ func TestGetJobLogData(t *testing.T) {
 	pool := &fakePool{}
 	c := NewController(pool)
 
-	if _, err := c.GetJobLogData("fake_ID"); err != nil {
+	if _, err := c.GetJobLogData("fake_ID"); err == nil {
-		if !errs.IsObjectNotFoundError(err) {
-			t.Errorf("expect object not found error but got '%s'\n", err)
-		}
-	} else {
 		t.Fatal("expect error but got nil")
 	}
 }

src/jobservice/job/impl/context.go

@@ -29,7 +29,6 @@ import (
 	"github.com/goharbor/harbor/src/jobservice/config"
 	"github.com/goharbor/harbor/src/jobservice/env"
 	"github.com/goharbor/harbor/src/jobservice/job"
-	jlogger "github.com/goharbor/harbor/src/jobservice/job/impl/logger"
 	"github.com/goharbor/harbor/src/jobservice/logger"
 	jmodel "github.com/goharbor/harbor/src/jobservice/models"
 )
@@ -124,11 +123,11 @@ func (c *Context) Build(dep env.JobData) (env.JobContext, error) {
 		jContext.properties[k] = v
 	}
 
-	// Init logger here
+	// Set loggers for job
-	logPath := fmt.Sprintf("%s/%s.log", config.GetLogBasePath(), dep.ID)
+	if err := setLoggers(func(lg logger.Interface) {
-	jContext.logger = jlogger.New(logPath, config.GetLogLevel())
+		jContext.logger = lg
-	if jContext.logger == nil {
+	}, dep.ID); err != nil {
-		return nil, errors.New("failed to initialize job logger")
+		return nil, err
 	}
 
 	if opCommandFunc, ok := dep.ExtraData["opCommandFunc"]; ok {
@@ -227,3 +226,37 @@ func getDBFromConfig(cfg map[string]interface{}) *models.Database {
 
 	return database
 }
+
+// create loggers based on the configurations and set it to the job executing context.
+func setLoggers(setter func(lg logger.Interface), jobID string) error {
+	if setter == nil {
+		return errors.New("missing setter func")
+	}
+
+	// Init job loggers here
+	lOptions := []logger.Option{}
+	for _, lc := range config.DefaultConfig.JobLoggerConfigs {
+		if lc.Name == logger.LoggerNameFile {
+			// Need extra param
+			fSettings := map[string]interface{}{}
+			for k, v := range lc.Settings {
+				// Copy settings
+				fSettings[k] = v
+			}
+			// Append file name param
+			fSettings["filename"] = fmt.Sprintf("%s.log", jobID)
+			lOptions = append(lOptions, logger.BackendOption(lc.Name, lc.Level, fSettings))
+		} else {
+			lOptions = append(lOptions, logger.BackendOption(lc.Name, lc.Level, lc.Settings))
+		}
+	}
+	// Get logger for the job
+	lg, err := logger.GetLogger(lOptions...)
+	if err != nil {
+		return fmt.Errorf("initialize job logger error: %s", err)
+	}
+
+	setter(lg)
+
+	return nil
+}

src/jobservice/job/impl/default_context.go

@@ -17,13 +17,10 @@ package impl
 import (
 	"context"
 	"errors"
-	"fmt"
 	"reflect"
 
-	"github.com/goharbor/harbor/src/jobservice/config"
 	"github.com/goharbor/harbor/src/jobservice/env"
 	"github.com/goharbor/harbor/src/jobservice/job"
-	jlogger "github.com/goharbor/harbor/src/jobservice/job/impl/logger"
 	"github.com/goharbor/harbor/src/jobservice/logger"
 	jmodel "github.com/goharbor/harbor/src/jobservice/models"
 )
@@ -72,11 +69,11 @@ func (c *DefaultContext) Build(dep env.JobData) (env.JobContext, error) {
 		}
 	}
 
-	// Init logger here
+	// Set loggers for job
-	logPath := fmt.Sprintf("%s/%s.log", config.GetLogBasePath(), dep.ID)
+	if err := setLoggers(func(lg logger.Interface) {
-	jContext.logger = jlogger.New(logPath, config.GetLogLevel())
+		jContext.logger = lg
-	if jContext.logger == nil {
+	}, dep.ID); err != nil {
-		return nil, errors.New("failed to initialize job logger")
+		return nil, err
 	}
 
 	if opCommandFunc, ok := dep.ExtraData["opCommandFunc"]; ok {

src/jobservice/job/impl/default_context_test.go

@@ -57,15 +57,19 @@ func TestDefaultContext(t *testing.T) {
 	jobData.ExtraData["checkInFunc"] = checkInFunc
 	jobData.ExtraData["launchJobFunc"] = launchJobFunc
 
-	oldLogConfig := config.DefaultConfig.LoggerConfig
+	oldLogConfig := config.DefaultConfig.JobLoggerConfigs
 	defer func() {
-		config.DefaultConfig.LoggerConfig = oldLogConfig
+		config.DefaultConfig.JobLoggerConfigs = oldLogConfig
 	}()
 
-	config.DefaultConfig.LoggerConfig = &config.LoggerConfig{
+	logSettings := map[string]interface{}{}
-		LogLevel:      "debug",
+	logSettings["base_dir"] = os.TempDir()
-		ArchivePeriod: 1,
+	config.DefaultConfig.JobLoggerConfigs = []*config.LoggerConfig{
-		BasePath:      os.TempDir(),
+		{
+			Level:    "DEBUG",
+			Name:     "FILE",
+			Settings: logSettings,
+		},
 	}
 
 	newJobContext, err := defaultContext.Build(jobData)

src/jobservice/job/impl/logger/job_logger.go

@@ -1,128 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package logger
-
-import (
-	"os"
-	"strings"
-
-	"github.com/goharbor/harbor/src/common/utils/log"
-	"github.com/goharbor/harbor/src/jobservice/logger"
-)
-
-// JobLogger is an implementation of logger.Interface.
-// It used in the job to output logs to the logfile.
-type JobLogger struct {
-	backendLogger *log.Logger
-	streamRef     *os.File
-}
-
-// New logger
-// nil might be returned
-func New(logPath string, level string) logger.Interface {
-	f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY, 0644)
-	if err != nil {
-		logger.Errorf("Failed to create job logger: %s", err)
-		return nil
-	}
-	logLevel := parseLevel(level)
-	backendLogger := log.New(f, log.NewTextFormatter(), logLevel)
-
-	return &JobLogger{
-		backendLogger: backendLogger,
-		streamRef:     f,
-	}
-}
-
-// Close the opened io stream
-// Implements logger.Closer interface
-func (jl *JobLogger) Close() error {
-	if jl.streamRef != nil {
-		return jl.streamRef.Close()
-	}
-
-	return nil
-}
-
-// Debug ...
-func (jl *JobLogger) Debug(v ...interface{}) {
-	jl.backendLogger.Debug(v...)
-}
-
-// Debugf with format
-func (jl *JobLogger) Debugf(format string, v ...interface{}) {
-	jl.backendLogger.Debugf(format, v...)
-}
-
-// Info ...
-func (jl *JobLogger) Info(v ...interface{}) {
-	jl.backendLogger.Info(v...)
-}
-
-// Infof with format
-func (jl *JobLogger) Infof(format string, v ...interface{}) {
-	jl.backendLogger.Infof(format, v...)
-}
-
-// Warning ...
-func (jl *JobLogger) Warning(v ...interface{}) {
-	jl.backendLogger.Warning(v...)
-}
-
-// Warningf with format
-func (jl *JobLogger) Warningf(format string, v ...interface{}) {
-	jl.backendLogger.Warningf(format, v...)
-}
-
-// Error ...
-func (jl *JobLogger) Error(v ...interface{}) {
-	jl.backendLogger.Error(v...)
-}
-
-// Errorf with format
-func (jl *JobLogger) Errorf(format string, v ...interface{}) {
-	jl.backendLogger.Errorf(format, v...)
-}
-
-// Fatal error
-func (jl *JobLogger) Fatal(v ...interface{}) {
-	jl.backendLogger.Fatal(v...)
-}
-
-// Fatalf error
-func (jl *JobLogger) Fatalf(format string, v ...interface{}) {
-	jl.backendLogger.Fatalf(format, v...)
-}
-
-func parseLevel(lvl string) log.Level {
-
-	var level = log.WarningLevel
-
-	switch strings.ToLower(lvl) {
-	case "debug":
-		level = log.DebugLevel
-	case "info":
-		level = log.InfoLevel
-	case "warning":
-		level = log.WarningLevel
-	case "error":
-		level = log.ErrorLevel
-	case "fatal":
-		level = log.FatalLevel
-	default:
-	}
-
-	return level
-}

src/jobservice/job/impl/logger/service_logger.go

@@ -1,88 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package logger
-
-import (
-	"os"
-
-	"github.com/goharbor/harbor/src/common/utils/log"
-)
-
-// ServiceLogger is an implementation of logger.Interface.
-// It used to log info in workerpool components.
-type ServiceLogger struct {
-	backendLogger *log.Logger
-}
-
-// NewServiceLogger to create new logger for job service
-// nil might be returned
-func NewServiceLogger(level string) *ServiceLogger {
-	logLevel := parseLevel(level)
-	backendLogger := log.New(os.Stdout, log.NewTextFormatter(), logLevel)
-
-	return &ServiceLogger{
-		backendLogger: backendLogger,
-	}
-}
-
-// Debug ...
-func (sl *ServiceLogger) Debug(v ...interface{}) {
-	sl.backendLogger.Debug(v...)
-}
-
-// Debugf with format
-func (sl *ServiceLogger) Debugf(format string, v ...interface{}) {
-	sl.backendLogger.Debugf(format, v...)
-}
-
-// Info ...
-func (sl *ServiceLogger) Info(v ...interface{}) {
-	sl.backendLogger.Info(v...)
-}
-
-// Infof with format
-func (sl *ServiceLogger) Infof(format string, v ...interface{}) {
-	sl.backendLogger.Infof(format, v...)
-}
-
-// Warning ...
-func (sl *ServiceLogger) Warning(v ...interface{}) {
-	sl.backendLogger.Warning(v...)
-}
-
-// Warningf with format
-func (sl *ServiceLogger) Warningf(format string, v ...interface{}) {
-	sl.backendLogger.Warningf(format, v...)
-}
-
-// Error ...
-func (sl *ServiceLogger) Error(v ...interface{}) {
-	sl.backendLogger.Error(v...)
-}
-
-// Errorf with format
-func (sl *ServiceLogger) Errorf(format string, v ...interface{}) {
-	sl.backendLogger.Errorf(format, v...)
-}
-
-// Fatal error
-func (sl *ServiceLogger) Fatal(v ...interface{}) {
-	sl.backendLogger.Fatal(v...)
-}
-
-// Fatalf error
-func (sl *ServiceLogger) Fatalf(format string, v ...interface{}) {
-	sl.backendLogger.Fatalf(format, v...)
-}

src/jobservice/logger/backend/file_logger.go

@@ -0,0 +1,90 @@
+package backend
+
+import (
+	"os"
+
+	"github.com/goharbor/harbor/src/common/utils/log"
+)
+
+// FileLogger is an implementation of logger.Interface.
+// It outputs logs to the specified logfile.
+type FileLogger struct {
+	backendLogger *log.Logger
+	streamRef     *os.File
+}
+
+// NewFileLogger crates a new file logger
+// nil might be returned
+func NewFileLogger(level string, logPath string) (*FileLogger, error) {
+	f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, err
+	}
+	logLevel := parseLevel(level)
+	backendLogger := log.New(f, log.NewTextFormatter(), logLevel)
+
+	return &FileLogger{
+		backendLogger: backendLogger,
+		streamRef:     f,
+	}, nil
+}
+
+// Close the opened io stream
+// Implements logger.Closer interface
+func (fl *FileLogger) Close() error {
+	if fl.streamRef != nil {
+		return fl.streamRef.Close()
+	}
+
+	return nil
+}
+
+// Debug ...
+func (fl *FileLogger) Debug(v ...interface{}) {
+	fl.backendLogger.Debug(v...)
+}
+
+// Debugf with format
+func (fl *FileLogger) Debugf(format string, v ...interface{}) {
+	fl.backendLogger.Debugf(format, v...)
+}
+
+// Info ...
+func (fl *FileLogger) Info(v ...interface{}) {
+	fl.backendLogger.Info(v...)
+}
+
+// Infof with format
+func (fl *FileLogger) Infof(format string, v ...interface{}) {
+	fl.backendLogger.Infof(format, v...)
+}
+
+// Warning ...
+func (fl *FileLogger) Warning(v ...interface{}) {
+	fl.backendLogger.Warning(v...)
+}
+
+// Warningf with format
+func (fl *FileLogger) Warningf(format string, v ...interface{}) {
+	fl.backendLogger.Warningf(format, v...)
+}
+
+// Error ...
+func (fl *FileLogger) Error(v ...interface{}) {
+	fl.backendLogger.Error(v...)
+}
+
+// Errorf with format
+func (fl *FileLogger) Errorf(format string, v ...interface{}) {
+	fl.backendLogger.Errorf(format, v...)
+}
+
+// Fatal error
+func (fl *FileLogger) Fatal(v ...interface{}) {
+	fl.backendLogger.Fatal(v...)
+}
+
+// Fatalf error
+func (fl *FileLogger) Fatalf(format string, v ...interface{}) {
+	fl.backendLogger.Fatalf(format, v...)
+}

src/jobservice/logger/backend/std_logger.go

@@ -0,0 +1,84 @@
+package backend
+
+import (
+	"os"
+
+	"github.com/goharbor/harbor/src/common/utils/log"
+)
+
+const (
+	// StdOut represents os.Stdout
+	StdOut = "std_out"
+	// StdErr represents os.StdErr
+	StdErr = "std_err"
+)
+
+// StdOutputLogger is an implementation of logger.Interface.
+// It outputs the log to the stdout/stderr.
+type StdOutputLogger struct {
+	backendLogger *log.Logger
+}
+
+// NewStdOutputLogger creates a new std output logger
+func NewStdOutputLogger(level string, output string) *StdOutputLogger {
+	logLevel := parseLevel(level)
+	logStream := os.Stdout
+	if output == StdErr {
+		logStream = os.Stderr
+	}
+	backendLogger := log.New(logStream, log.NewTextFormatter(), logLevel)
+
+	return &StdOutputLogger{
+		backendLogger: backendLogger,
+	}
+}
+
+// Debug ...
+func (sl *StdOutputLogger) Debug(v ...interface{}) {
+	sl.backendLogger.Debug(v...)
+}
+
+// Debugf with format
+func (sl *StdOutputLogger) Debugf(format string, v ...interface{}) {
+	sl.backendLogger.Debugf(format, v...)
+}
+
+// Info ...
+func (sl *StdOutputLogger) Info(v ...interface{}) {
+	sl.backendLogger.Info(v...)
+}
+
+// Infof with format
+func (sl *StdOutputLogger) Infof(format string, v ...interface{}) {
+	sl.backendLogger.Infof(format, v...)
+}
+
+// Warning ...
+func (sl *StdOutputLogger) Warning(v ...interface{}) {
+	sl.backendLogger.Warning(v...)
+}
+
+// Warningf with format
+func (sl *StdOutputLogger) Warningf(format string, v ...interface{}) {
+	sl.backendLogger.Warningf(format, v...)
+}
+
+// Error ...
+func (sl *StdOutputLogger) Error(v ...interface{}) {
+	sl.backendLogger.Error(v...)
+}
+
+// Errorf with format
+func (sl *StdOutputLogger) Errorf(format string, v ...interface{}) {
+	sl.backendLogger.Errorf(format, v...)
+}
+
+// Fatal error
+func (sl *StdOutputLogger) Fatal(v ...interface{}) {
+	sl.backendLogger.Fatal(v...)
+}
+
+// Fatalf error
+func (sl *StdOutputLogger) Fatalf(format string, v ...interface{}) {
+	sl.backendLogger.Fatalf(format, v...)
+}

src/jobservice/logger/backend/utils.go

@@ -0,0 +1,28 @@
+package backend
+
+import (
+	"strings"
+
+	"github.com/goharbor/harbor/src/common/utils/log"
+)
+
+func parseLevel(lvl string) log.Level {
+
+	var level = log.WarningLevel
+
+	switch strings.ToLower(lvl) {
+	case "debug":
+		level = log.DebugLevel
+	case "info":
+		level = log.InfoLevel
+	case "warning":
+		level = log.WarningLevel
+	case "error":
+		level = log.ErrorLevel
+	case "fatal":
+		level = log.FatalLevel
+	default:
+	}
+
+	return level
+}

src/jobservice/logger/base.go

@@ -0,0 +1,224 @@
+package logger
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"sync"
+
+	"github.com/goharbor/harbor/src/jobservice/config"
+	"github.com/goharbor/harbor/src/jobservice/logger/getter"
+	"github.com/goharbor/harbor/src/jobservice/logger/sweeper"
+)
+
+var singletons sync.Map
+
+// GetLogger gets an unified logger entry for logging per the passed settings.
+// The logger may built based on the multiple registered logger backends.
+//
+//  loggerOptions ...Option : logger options
+//
+// If failed, a nil logger and a non-nil error will be returned.
+// Otherwise, a non nil logger is returned with nil error.
+func GetLogger(loggerOptions ...Option) (Interface, error) {
+	lOptions := &options{
+		values: make(map[string][]OptionItem),
+	}
+
+	// Config
+	for _, op := range loggerOptions {
+		op.Apply(lOptions)
+	}
+
+	// No options specified, enable std as default
+	if len(loggerOptions) == 0 {
+		defaultOp := BackendOption(LoggerNameStdOutput, "", nil)
+		defaultOp.Apply(lOptions)
+	}
+
+	// Create backends
+	loggers := []Interface{}
+	for name, ops := range lOptions.values {
+		if !IsKnownLogger(name) {
+			return nil, fmt.Errorf("no logger registered for name '%s'", name)
+		}
+
+		d := KnownLoggers(name)
+		var (
+			l  Interface
+			ok bool
+		)
+
+		// Singleton
+		if d.Singleton {
+			var li interface{}
+			li, ok = singletons.Load(name)
+			if ok {
+				l = li.(Interface)
+			}
+		}
+
+		if !ok {
+			var err error
+			l, err = d.Logger(ops...)
+			if err != nil {
+				return nil, err
+			}
+
+			// Cache it
+			singletons.Store(name, l)
+		}
+
+		// Append to the logger list as logger entry backends
+		loggers = append(loggers, l)
+	}
+
+	return NewEntry(loggers), nil
+}
+
+// GetSweeper gets an unified sweeper controller for sweeping purpose.
+//
+// context context.Context  : system contex used to control the sweeping loops
+// sweeperOptions ...Option : sweeper options
+//
+// If failed, a nil sweeper and a non-nil error will be returned.
+// Otherwise, a non nil sweeper is returned with nil error.
+func GetSweeper(context context.Context, sweeperOptions ...Option) (sweeper.Interface, error) {
+	// No default sweeper will provdie
+	// If no one is configured, directly return nil with error
+	if len(sweeperOptions) == 0 {
+		return nil, errors.New("no options provided for creating sweeper controller")
+	}
+
+	sOptions := &options{
+		values: make(map[string][]OptionItem),
+	}
+
+	// Config
+	for _, op := range sweeperOptions {
+		op.Apply(sOptions)
+	}
+
+	sweepers := []sweeper.Interface{}
+	for name, ops := range sOptions.values {
+		if !HasSweeper(name) {
+			return nil, fmt.Errorf("no sweeper provided for the logger %s", name)
+		}
+
+		d := KnownLoggers(name)
+		s, err := d.Sweeper(ops...)
+		if err != nil {
+			return nil, err
+		}
+
+		sweepers = append(sweepers, s)
+	}
+
+	return NewSweeperController(context, sweepers), nil
+}
+
+// GetLogDataGetter return the 1st matched log data getter interface
+//
+//  loggerOptions ...Option : logger options
+//
+// If failed,
+//   configured but initialize failed: a nil log data getter and a non-nil error will be returned.
+//   no getter configured: a nil log data getter with a nil error are returned
+// Otherwise, a non nil log data getter is returned with nil error.
+func GetLogDataGetter(loggerOptions ...Option) (getter.Interface, error) {
+	if len(loggerOptions) == 0 {
+		// If no options, directly return nil interface with error
+		return nil, errors.New("no options provided to create log data getter")
+	}
+
+	lOptions := &options{
+		values: make(map[string][]OptionItem),
+	}
+
+	// Config
+	for _, op := range loggerOptions {
+		op.Apply(lOptions)
+	}
+
+	// Iterate with specified order
+	keys := []string{}
+	for k := range lOptions.values {
+		keys = append(keys, k)
+	}
+
+	// Sort
+	sort.Strings(keys)
+
+	for _, k := range keys {
+		if HasGetter(k) {
+			// 1st match
+			d := knownLoggers[k]
+			theGetter, err := d.Getter(lOptions.values[k]...)
+			if err != nil {
+				return nil, err
+			}
+
+			return theGetter, nil
+		}
+	}
+
+	// No one configured
+	return nil, nil
+}
+
+// Init the loggers and sweepers
+func Init(ctx context.Context) error {
+	// For loggers
+	options := []Option{}
+	// For sweepers
+	sOptions := []Option{}
+
+	for _, lc := range config.DefaultConfig.LoggerConfigs {
+		options = append(options, BackendOption(lc.Name, lc.Level, lc.Settings))
+		if lc.Sweeper != nil {
+			sOptions = append(sOptions, SweeperOption(lc.Name, lc.Sweeper.Duration, lc.Sweeper.Settings))
+		}
+	}
+
+	// Get loggers for job service
+	lg, err := GetLogger(options...)
+	if err != nil {
+		return err
+	}
+	jobServiceLogger = lg
+
+	jOptions := []Option{}
+	// Append configured sweepers in job loggers if existing
+	for _, lc := range config.DefaultConfig.JobLoggerConfigs {
+		jOptions = append(jOptions, BackendOption(lc.Name, lc.Level, lc.Settings))
+		if lc.Sweeper != nil {
+			sOptions = append(sOptions, SweeperOption(lc.Name, lc.Sweeper.Duration, lc.Sweeper.Settings))
+		}
+	}
+
+	// Get log data getter with the same options of job loggers
+	g, err := GetLogDataGetter(jOptions...)
+	if err != nil {
+		return err
+	}
+	if g != nil {
+		logDataGetter = g
+	}
+
+	// If sweepers configured
+	if len(sOptions) > 0 {
+		// Get the sweeper controller
+		sweeper, err := GetSweeper(ctx, sOptions...)
+		if err != nil {
+			return fmt.Errorf("create logger sweeper error: %s", err)
+		}
+		// Start sweep loop
+		_, err = sweeper.Sweep()
+		if err != nil {
+			return fmt.Errorf("start logger sweeper error: %s", err)
+		}
+	}
+
+	return nil
+}

src/jobservice/logger/entry.go

@@ -0,0 +1,84 @@
+package logger
+
+// Entry provides unique interfaces on top of multiple logger backends.
+// Entry also implements @Interface.
+type Entry struct {
+	loggers []Interface
+}
+
+// NewEntry creates a new logger Entry
+func NewEntry(loggers []Interface) *Entry {
+	return &Entry{
+		loggers: loggers,
+	}
+}
+
+// Debug ...
+func (e *Entry) Debug(v ...interface{}) {
+	for _, l := range e.loggers {
+		l.Debug(v...)
+	}
+}
+
+// Debugf with format
+func (e *Entry) Debugf(format string, v ...interface{}) {
+	for _, l := range e.loggers {
+		l.Debugf(format, v...)
+	}
+}
+
+// Info ...
+func (e *Entry) Info(v ...interface{}) {
+	for _, l := range e.loggers {
+		l.Info(v...)
+	}
+}
+
+// Infof with format
+func (e *Entry) Infof(format string, v ...interface{}) {
+	for _, l := range e.loggers {
+		l.Infof(format, v...)
+	}
+}
+
+// Warning ...
+func (e *Entry) Warning(v ...interface{}) {
+	for _, l := range e.loggers {
+		l.Warning(v...)
+	}
+}
+
+// Warningf with format
+func (e *Entry) Warningf(format string, v ...interface{}) {
+	for _, l := range e.loggers {
+		l.Warningf(format, v...)
+	}
+}
+
+// Error ...
+func (e *Entry) Error(v ...interface{}) {
+	for _, l := range e.loggers {
+		l.Error(v...)
+	}
+}
+
+// Errorf with format
+func (e *Entry) Errorf(format string, v ...interface{}) {
+	for _, l := range e.loggers {
+		l.Errorf(format, v...)
+	}
+}
+
+// Fatal error
+func (e *Entry) Fatal(v ...interface{}) {
+	for _, l := range e.loggers {
+		l.Fatal(v...)
+	}
+}
+
+// Fatalf error
+func (e *Entry) Fatalf(format string, v ...interface{}) {
+	for _, l := range e.loggers {
+		l.Fatalf(format, v...)
+	}
+}

src/jobservice/logger/factory.go

@@ -0,0 +1,54 @@
+package logger
+
+import (
+	"errors"
+	"path"
+
+	"github.com/goharbor/harbor/src/jobservice/logger/backend"
+)
+
+// Factory creates a new logger based on the settings.
+type Factory func(options ...OptionItem) (Interface, error)
+
+// FileFactory is factory of file logger
+func FileFactory(options ...OptionItem) (Interface, error) {
+	var level, baseDir, fileName string
+	for _, op := range options {
+		switch op.Field() {
+		case "level":
+			level = op.String()
+		case "base_dir":
+			baseDir = op.String()
+		case "filename":
+			fileName = op.String()
+		default:
+
+		}
+	}
+
+	if len(baseDir) == 0 {
+		return nil, errors.New("missing base dir option of the file logger")
+	}
+
+	if len(fileName) == 0 {
+		return nil, errors.New("missing file name option of the file logger")
+	}
+
+	return backend.NewFileLogger(level, path.Join(baseDir, fileName))
+}
+
+// StdFactory is factory of std output logger.
+func StdFactory(options ...OptionItem) (Interface, error) {
+	var level, output string
+	for _, op := range options {
+		switch op.Field() {
+		case "level":
+			level = op.String()
+		case "output":
+			output = op.String()
+		default:
+		}
+	}
+
+	return backend.NewStdOutputLogger(level, output), nil
+}

src/jobservice/logger/getter/Interface.go

@@ -0,0 +1,12 @@
+package getter
+
+// Interface defines operations of a log data getter
+type Interface interface {
+	// Retrieve the log data of the specified log entry
+	//
+	// logID string : the id of the log entry. e.g: file name a.log for file log
+	//
+	// If succeed, log data bytes will be returned
+	// otherwise, a non nil error is returned
+	Retrieve(logID string) ([]byte, error)
+}

src/jobservice/logger/getter/file_getter.go

@@ -0,0 +1,37 @@
+package getter
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"path"
+
+	"github.com/goharbor/harbor/src/jobservice/errs"
+
+	"github.com/goharbor/harbor/src/jobservice/utils"
+)
+
+// FileGetter is responsible for retrieving file log data
+type FileGetter struct {
+	baseDir string
+}
+
+// NewFileGetter is constructor of FileGetter
+func NewFileGetter(baseDir string) *FileGetter {
+	return &FileGetter{baseDir}
+}
+
+// Retrieve implements @Interface.Retrieve
+func (fg *FileGetter) Retrieve(logID string) ([]byte, error) {
+	if len(logID) == 0 {
+		return nil, errors.New("empty log identify")
+	}
+
+	fPath := path.Join(fg.baseDir, fmt.Sprintf("%s.log", logID))
+
+	if !utils.FileExists(fPath) {
+		return nil, errs.NoObjectFoundError(logID)
+	}
+
+	return ioutil.ReadFile(fPath)
+}

src/jobservice/logger/getter_factory.go

@@ -0,0 +1,27 @@
+package logger
+
+import (
+	"errors"
+
+	"github.com/goharbor/harbor/src/jobservice/logger/getter"
+)
+
+// GetterFactory is responsible for creating a log data getter based on the options
+type GetterFactory func(options ...OptionItem) (getter.Interface, error)
+
+// FileGetterFactory creates a getter for the "FILE" logger
+func FileGetterFactory(options ...OptionItem) (getter.Interface, error) {
+	var baseDir string
+	for _, op := range options {
+		if op.Field() == "base_dir" {
+			baseDir = op.String()
+			break
+		}
+	}
+
+	if len(baseDir) == 0 {
+		return nil, errors.New("missing required option 'base_dir'")
+	}
+
+	return getter.NewFileGetter(baseDir), nil
+}

src/jobservice/logger/known_loggers.go

@@ -0,0 +1,81 @@
+package logger
+
+import "strings"
+
+const (
+	// LoggerNameFile is unique name of the file logger.
+	LoggerNameFile = "FILE"
+	// LoggerNameStdOutput is the unique name of the std logger.
+	LoggerNameStdOutput = "STD_OUTPUT"
+)
+
+// Declaration is used to declare a supported logger.
+// Use this declaration to indicate what logger and sweeper will be provided.
+type Declaration struct {
+	Logger  Factory
+	Sweeper SweeperFactory
+	Getter  GetterFactory
+	// Indicate if the logger is a singleton logger
+	Singleton bool
+}
+
+// knownLoggers is a static logger registry.
+// All the implemented loggers (w/ sweeper) should be registered
+// with an unique name in this registry. Then they can be used to
+// log info.
+var knownLoggers = map[string]*Declaration{
+	// File logger
+	LoggerNameFile: {FileFactory, FileSweeperFactory, FileGetterFactory, false},
+	// STD output(both stdout and stderr) logger
+	LoggerNameStdOutput: {StdFactory, nil, nil, true},
+}
+
+// IsKnownLogger checks if the logger is supported with name.
+func IsKnownLogger(name string) bool {
+	_, ok := knownLoggers[name]
+
+	return ok
+}
+
+// HasSweeper checks if the logger with the name provides a sweeper.
+func HasSweeper(name string) bool {
+	d, ok := knownLoggers[name]
+
+	return ok && d.Sweeper != nil
+}
+
+// HasGetter checks if the logger with the name provides a log data getter.
+func HasGetter(name string) bool {
+	d, ok := knownLoggers[name]
+
+	return ok && d.Getter != nil
+}
+
+// KnownLoggers return the declaration by the name
+func KnownLoggers(name string) *Declaration {
+	return knownLoggers[name]
+}
+
+// All known levels which are supported.
+var debugLevels = []string{
+	"DEBUG",
+	"INFO",
+	"WARNING",
+	"ERROR",
+	"FATAL",
+}
+
+// IsKnownLevel is used to check if the logger level is supported.
+func IsKnownLevel(level string) bool {
+	if len(level) == 0 {
+		return false
+	}
+
+	for _, lvl := range debugLevels {
+		if lvl == strings.ToUpper(level) {
+			return true
+		}
+	}
+
+	return false
+}

src/jobservice/logger/log_data_handler.go

@@ -0,0 +1,23 @@
+package logger
+
+import (
+	"errors"
+
+	"github.com/goharbor/harbor/src/jobservice/logger/getter"
+)
+
+var logDataGetter getter.Interface
+
+// Retrieve is wrapper func for getter.Retrieve
+func Retrieve(logID string) ([]byte, error) {
+	if logDataGetter == nil {
+		return nil, errors.New("no log data getter is configured")
+	}
+
+	return logDataGetter.Retrieve(logID)
+}
+
+// HasLogGetterConfigured checks if a log data getter is there for using
+func HasLogGetterConfigured() bool {
+	return logDataGetter != nil
+}

src/jobservice/logger/options.go

@@ -0,0 +1,99 @@
+package logger
+
+// options keep settings for loggers/sweepers
+// Indexed by the logger unique name
+type options struct {
+	values map[string][]OptionItem
+}
+
+// Option represents settings of the logger
+type Option struct {
+	// Apply logger option
+	Apply func(op *options)
+}
+
+// BackendOption creates option for the specified backend.
+func BackendOption(name string, level string, settings map[string]interface{}) Option {
+	return Option{func(op *options) {
+		vals := make([]OptionItem, 0)
+		vals = append(vals, OptionItem{"level", level})
+
+		// Append extra settings if existing
+		if len(settings) > 0 {
+			for k, v := range settings {
+				vals = append(vals, OptionItem{k, v})
+			}
+		}
+
+		// Append with overriding way
+		op.values[name] = vals
+	}}
+}
+
+// SweeperOption creates option for the sweeper.
+func SweeperOption(name string, duration int, settings map[string]interface{}) Option {
+	return Option{func(op *options) {
+		vals := make([]OptionItem, 0)
+		vals = append(vals, OptionItem{"duration", duration})
+
+		// Append settings if existing
+		if len(settings) > 0 {
+			for k, v := range settings {
+				vals = append(vals, OptionItem{k, v})
+			}
+		}
+
+		// Append with overriding way
+		op.values[name] = vals
+	}}
+}
+
+// GetterOption creates option for the getter.
+func GetterOption(name string, settings map[string]interface{}) Option {
+	return Option{func(op *options) {
+		vals := make([]OptionItem, 0)
+		// Append settings if existing
+		if len(settings) > 0 {
+			for k, v := range settings {
+				vals = append(vals, OptionItem{k, v})
+			}
+		}
+
+		// Append with overriding way
+		op.values[name] = vals
+	}}
+}
+
+// OptionItem is a simple wrapper of property and value
+type OptionItem struct {
+	field string
+	val   interface{}
+}
+
+// Field returns name of the option
+func (o *OptionItem) Field() string {
+	return o.field
+}
+
+// Int returns the integer value of option
+func (o *OptionItem) Int() int {
+	if o.val == nil {
+		return 0
+	}
+
+	return o.val.(int)
+}
+
+// String returns the string value of option
+func (o *OptionItem) String() string {
+	if o.val == nil {
+		return ""
+	}
+
+	return o.val.(string)
+}
+
+// Raw returns the raw value
+func (o *OptionItem) Raw() interface{} {
+	return o.val
+}

src/jobservice/logger/service.go

@@ -18,18 +18,13 @@ import (
 	"log"
 )
 
-// sLogger is used to log for workerpool itself
+// jobServiceLogger is used to log for job service itself
-var sLogger Interface
+var jobServiceLogger Interface
-
-// SetLogger sets the logger implementation
-func SetLogger(logger Interface) {
-	sLogger = logger
-}
 
 // Debug ...
 func Debug(v ...interface{}) {
-	if sLogger != nil {
+	if jobServiceLogger != nil {
-		sLogger.Debug(v...)
+		jobServiceLogger.Debug(v...)
 		return
 	}
 
@@ -38,8 +33,8 @@ func Debug(v ...interface{}) {
 
 // Debugf for debuging with format
 func Debugf(format string, v ...interface{}) {
-	if sLogger != nil {
+	if jobServiceLogger != nil {
-		sLogger.Debugf(format, v...)
+		jobServiceLogger.Debugf(format, v...)
 		return
 	}
 
@@ -48,8 +43,8 @@ func Debugf(format string, v ...interface{}) {
 
 // Info ...
 func Info(v ...interface{}) {
-	if sLogger != nil {
+	if jobServiceLogger != nil {
-		sLogger.Info(v...)
+		jobServiceLogger.Info(v...)
 		return
 	}
 
@@ -58,8 +53,8 @@ func Info(v ...interface{}) {
 
 // Infof for logging info with format
 func Infof(format string, v ...interface{}) {
-	if sLogger != nil {
+	if jobServiceLogger != nil {
-		sLogger.Infof(format, v...)
+		jobServiceLogger.Infof(format, v...)
 		return
 	}
 
@@ -68,8 +63,8 @@ func Infof(format string, v ...interface{}) {
 
 // Warning ...
 func Warning(v ...interface{}) {
-	if sLogger != nil {
+	if jobServiceLogger != nil {
-		sLogger.Warning(v...)
+		jobServiceLogger.Warning(v...)
 		return
 	}
 
@@ -78,8 +73,8 @@ func Warning(v ...interface{}) {
 
 // Warningf for warning with format
 func Warningf(format string, v ...interface{}) {
-	if sLogger != nil {
+	if jobServiceLogger != nil {
-		sLogger.Warningf(format, v...)
+		jobServiceLogger.Warningf(format, v...)
 		return
 	}
 
@@ -88,8 +83,8 @@ func Warningf(format string, v ...interface{}) {
 
 // Error for logging error
 func Error(v ...interface{}) {
-	if sLogger != nil {
+	if jobServiceLogger != nil {
-		sLogger.Error(v...)
+		jobServiceLogger.Error(v...)
 		return
 	}
 
@@ -98,8 +93,8 @@ func Error(v ...interface{}) {
 
 // Errorf for logging error with format
 func Errorf(format string, v ...interface{}) {
-	if sLogger != nil {
+	if jobServiceLogger != nil {
-		sLogger.Errorf(format, v...)
+		jobServiceLogger.Errorf(format, v...)
 		return
 	}
 
@@ -108,8 +103,8 @@ func Errorf(format string, v ...interface{}) {
 
 // Fatal ...
 func Fatal(v ...interface{}) {
-	if sLogger != nil {
+	if jobServiceLogger != nil {
-		sLogger.Fatal(v...)
+		jobServiceLogger.Fatal(v...)
 		return
 	}
 
@@ -118,8 +113,8 @@ func Fatal(v ...interface{}) {
 
 // Fatalf for fatal error with error
 func Fatalf(format string, v ...interface{}) {
-	if sLogger != nil {
+	if jobServiceLogger != nil {
-		sLogger.Fatalf(format, v...)
+		jobServiceLogger.Fatalf(format, v...)
 		return
 	}
 

src/jobservice/logger/service_logger_test.go

@@ -1,87 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package logger
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestServiceLogger(t *testing.T) {
-	testingLogger := &fakeLogger{}
-	SetLogger(testingLogger)
-
-	Debug("DEBUG")
-	Debugf("%s\n", "DEBUGF")
-	Info("INFO")
-	Infof("%s\n", "INFOF")
-	Warning("WARNING")
-	Warningf("%s\n", "WARNINGF")
-	Error("ERROR")
-	Errorf("%s\n", "ERRORF")
-	Fatal("FATAL")
-	Fatalf("%s\n", "FATALF")
-}
-
-type fakeLogger struct{}
-
-// For debuging
-func (fl *fakeLogger) Debug(v ...interface{}) {
-	fmt.Println(v...)
-}
-
-// For debuging with format
-func (fl *fakeLogger) Debugf(format string, v ...interface{}) {
-	fmt.Printf(format, v...)
-}
-
-// For logging info
-func (fl *fakeLogger) Info(v ...interface{}) {
-	fmt.Println(v...)
-}
-
-// For logging info with format
-func (fl *fakeLogger) Infof(format string, v ...interface{}) {
-	fmt.Printf(format, v...)
-}
-
-// For warning
-func (fl *fakeLogger) Warning(v ...interface{}) {
-	fmt.Println(v...)
-}
-
-// For warning with format
-func (fl *fakeLogger) Warningf(format string, v ...interface{}) {
-	fmt.Printf(format, v...)
-}
-
-// For logging error
-func (fl *fakeLogger) Error(v ...interface{}) {
-	fmt.Println(v...)
-}
-
-// For logging error with format
-func (fl *fakeLogger) Errorf(format string, v ...interface{}) {
-	fmt.Printf(format, v...)
-}
-
-// For fatal error
-func (fl *fakeLogger) Fatal(v ...interface{}) {
-	fmt.Println(v...)
-}
-
-// For fatal error with error
-func (fl *fakeLogger) Fatalf(format string, v ...interface{}) {
-	fmt.Printf(format, v...)
-}

src/jobservice/logger/sweeper.go

@@ -1,103 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package logger
-
-import (
-	"context"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"time"
-)
-
-const (
-	oneDay = 3600 * 24
-)
-
-// Sweeper takes charge of archive the outdated log files of jobs.
-type Sweeper struct {
-	context context.Context
-	workDir string
-	period  uint
-}
-
-// NewSweeper creates new prt of Sweeper
-func NewSweeper(ctx context.Context, workDir string, period uint) *Sweeper {
-	return &Sweeper{ctx, workDir, period}
-}
-
-// Start to work
-func (s *Sweeper) Start() {
-	go s.loop()
-	Info("Logger sweeper is started")
-}
-
-func (s *Sweeper) loop() {
-	// Apply default if needed before starting
-	if s.period == 0 {
-		s.period = 1
-	}
-
-	defer func() {
-		Info("Logger sweeper is stopped")
-	}()
-
-	// First run
-	go s.clear()
-
-	ticker := time.NewTicker(time.Duration(s.period*oneDay+5) * time.Second)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-s.context.Done():
-			return
-		case <-ticker.C:
-			go s.clear()
-		}
-	}
-}
-
-func (s *Sweeper) clear() {
-	var (
-		cleared uint
-		count   = &cleared
-	)
-
-	Info("Start to clear the job outdated log files")
-	defer func() {
-		Infof("%d job outdated log files cleared", *count)
-	}()
-
-	logFiles, err := ioutil.ReadDir(s.workDir)
-	if err != nil {
-		Errorf("Failed to get the outdated log files under '%s' with error: %s\n", s.workDir, err)
-		return
-	}
-	if len(logFiles) == 0 {
-		return
-	}
-
-	for _, logFile := range logFiles {
-		if logFile.ModTime().Add(time.Duration(s.period*oneDay) * time.Second).Before(time.Now()) {
-			logFilePath := fmt.Sprintf("%s%s%s", s.workDir, string(os.PathSeparator), logFile.Name())
-			if err := os.Remove(logFilePath); err == nil {
-				cleared++
-			} else {
-				Warningf("Failed to remove log file '%s'\n", logFilePath)
-			}
-		}
-	}
-}

src/jobservice/logger/sweeper/file_sweeper.go

@@ -0,0 +1,70 @@
+package sweeper
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+	"time"
+)
+
+const (
+	oneDay = 24 * time.Hour
+)
+
+// FileSweeper is used to sweep the file logs
+type FileSweeper struct {
+	duration int
+	workDir  string
+}
+
+// NewFileSweeper is constructor of FileSweeper
+func NewFileSweeper(workDir string, duration int) *FileSweeper {
+	return &FileSweeper{
+		workDir:  workDir,
+		duration: duration,
+	}
+}
+
+// Sweep logs
+func (fs *FileSweeper) Sweep() (int, error) {
+	cleared := 0
+
+	logFiles, err := ioutil.ReadDir(fs.workDir)
+	if err != nil {
+		return 0, fmt.Errorf("getting outdated log files under '%s' failed with error: %s", fs.workDir, err)
+	}
+
+	// Nothing to sweep
+	if len(logFiles) == 0 {
+		return 0, nil
+	}
+
+	// Start to sweep log files
+	// Record all errors
+	errs := make([]string, 0)
+	for _, logFile := range logFiles {
+		if logFile.ModTime().Add(time.Duration(fs.duration) * oneDay).Before(time.Now()) {
+			logFilePath := path.Join(fs.workDir, logFile.Name())
+			if err := os.Remove(logFilePath); err != nil {
+				errs = append(errs, fmt.Sprintf("remove log file '%s' error: %s", logFilePath, err))
+				continue // go on for next one
+			}
+
+			cleared++
+		}
+	}
+
+	if len(errs) > 0 {
+		err = fmt.Errorf("%s", strings.Join(errs, "\n"))
+	}
+
+	// Return error with high priority
+	return cleared, err
+}
+
+// Duration for sweeping
+func (fs *FileSweeper) Duration() int {
+	return fs.duration
+}

src/jobservice/logger/sweeper/interface.go

@@ -0,0 +1,13 @@
+package sweeper
+
+// Interface defines the operations a sweeper should have
+type Interface interface {
+	// Sweep the outdated log entries if necessary
+	//
+	// If failed, an non-nil error will return
+	// If succeeded, count of sweepped log entries is returned
+	Sweep() (int, error)
+
+	// Return the sweeping duration with day unit.
+	Duration() int
+}

src/jobservice/logger/sweeper_controller.go

@@ -0,0 +1,98 @@
+package logger
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"time"
+
+	"github.com/goharbor/harbor/src/jobservice/logger/sweeper"
+)
+
+const (
+	oneDay = 24 * time.Hour
+)
+
+// SweeperController is an unified sweeper entry and built on top of multiple sweepers.
+// It's responsible for starting the configured sweepers.
+type SweeperController struct {
+	context  context.Context
+	sweepers []sweeper.Interface
+	errChan  chan error
+}
+
+// NewSweeperController is constructor of controller.
+func NewSweeperController(ctx context.Context, sweepers []sweeper.Interface) *SweeperController {
+	return &SweeperController{
+		context:  ctx,
+		sweepers: sweepers,
+		errChan:  make(chan error, 1),
+	}
+}
+
+// Sweep logs
+func (c *SweeperController) Sweep() (int, error) {
+	// Start to process errors
+	go func() {
+		for {
+			select {
+			case err := <-c.errChan:
+				Error(err)
+			case <-c.context.Done():
+				return
+			}
+		}
+	}()
+
+	for _, s := range c.sweepers {
+		go func(sw sweeper.Interface) {
+			c.startSweeper(sw)
+		}(s)
+	}
+
+	return 0, nil
+}
+
+// Duration = -1 for controller
+func (c *SweeperController) Duration() int {
+	return -1
+}
+
+func (c *SweeperController) startSweeper(s sweeper.Interface) {
+	d := s.Duration()
+	if d <= 0 {
+		d = 1
+	}
+
+	// Use the type name as a simple ID
+	sid := reflect.TypeOf(s).String()
+	defer Infof("sweeper %s exit", sid)
+
+	// First run
+	go c.doSweeping(sid, s)
+
+	// Loop
+	ticker := time.NewTicker(time.Duration(d) * oneDay)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			go c.doSweeping(sid, s)
+		case <-c.context.Done():
+			return
+		}
+	}
+}
+
+func (c *SweeperController) doSweeping(sid string, s sweeper.Interface) {
+	Debugf("Sweeper %s is under working", sid)
+
+	count, err := s.Sweep()
+	if err != nil {
+		c.errChan <- fmt.Errorf("sweep logs error in %s at %d: %s", sid, time.Now().Unix(), err)
+		return
+	}
+
+	Infof("%d outdated log entries are sweepped by sweeper %s", count, sid)
+}

src/jobservice/logger/sweeper_factory.go

@@ -0,0 +1,32 @@
+package logger
+
+import (
+	"errors"
+
+	"github.com/goharbor/harbor/src/jobservice/logger/sweeper"
+)
+
+// SweeperFactory is responsible for creating a sweeper.Interface based on the settings
+type SweeperFactory func(options ...OptionItem) (sweeper.Interface, error)
+
+// FileSweeperFactory creates file sweeper.
+func FileSweeperFactory(options ...OptionItem) (sweeper.Interface, error) {
+	var workDir, duration = "", 1
+	for _, op := range options {
+		switch op.Field() {
+		case "work_dir":
+			workDir = op.String()
+		case "duration":
+			if op.Int() > 0 {
+				duration = op.Int()
+			}
+		default:
+		}
+	}
+
+	if len(workDir) == 0 {
+		return nil, errors.New("missing required option 'work_dir'")
+	}
+
+	return sweeper.NewFileSweeper(workDir, duration), nil
+}

src/jobservice/logger/sweeper_test.go

@@ -1,48 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package logger
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"testing"
-	"time"
-)
-
-func TestSweeper(t *testing.T) {
-	workDir := "/tmp/sweeper_logs"
-
-	if err := os.MkdirAll(workDir, 0755); err != nil {
-		t.Fatal(err)
-	}
-	_, err := os.Create(fmt.Sprintf("%s/sweeper_test.log", workDir))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	sweeper := NewSweeper(ctx, workDir, 1)
-	sweeper.Start()
-	<-time.After(100 * time.Millisecond)
-
-	if err := os.Remove(fmt.Sprintf("%s/sweeper_test.log", workDir)); err != nil {
-		t.Fatal(err)
-	}
-	if err := os.Remove(workDir); err != nil {
-		t.Fatal(err)
-	}
-}

src/jobservice/main.go

@@ -15,14 +15,11 @@
 package main
 
 import (
-	"errors"
+	"context"
 	"flag"
+	"fmt"
 
-	"github.com/goharbor/harbor/src/adminserver/client"
 	"github.com/goharbor/harbor/src/jobservice/config"
-	"github.com/goharbor/harbor/src/jobservice/env"
-	"github.com/goharbor/harbor/src/jobservice/job/impl"
-	ilogger "github.com/goharbor/harbor/src/jobservice/job/impl/logger"
 	"github.com/goharbor/harbor/src/jobservice/logger"
 	"github.com/goharbor/harbor/src/jobservice/runtime"
 	"github.com/goharbor/harbor/src/jobservice/utils"
@@ -36,16 +33,25 @@ func main() {
 	// Missing config file
 	if configPath == nil || utils.IsEmptyStr(*configPath) {
 		flag.Usage()
-		logger.Fatal("Config file should be specified")
+		panic("no config file is specified")
 	}
 
 	// Load configurations
 	if err := config.DefaultConfig.Load(*configPath, true); err != nil {
-		logger.Fatalf("Failed to load configurations with error: %s\n", err)
+		panic(fmt.Sprintf("load configurations error: %s\n", err))
+	}
+
+	// Create the root context
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Initialize logger
+	if err := logger.Init(ctx); err != nil {
+		panic(err)
 	}
 
 	// Set job context initializer
-	runtime.JobService.SetJobContextInitializer(func(ctx *env.Context) (env.JobContext, error) {
+	/*runtime.JobService.SetJobContextInitializer(func(ctx *env.Context) (env.JobContext, error) {
 		secret := config.GetAuthSecret()
 		if utils.IsEmptyStr(secret) {
 			return nil, errors.New("empty auth secret")
@@ -59,12 +65,8 @@ func main() {
 		}
 
 		return jobCtx, nil
-	})
+	})*/
-
-	// New logger for job service
-	sLogger := ilogger.NewServiceLogger(config.GetLogLevel())
-	logger.SetLogger(sLogger)
 
 	// Start
-	runtime.JobService.LoadAndRun()
+	runtime.JobService.LoadAndRun(ctx, cancel)
 }

src/jobservice/runtime/bootstrap.go

@@ -64,11 +64,7 @@ func (bs *Bootstrap) SetJobContextInitializer(initializer env.JobContextInitiali
 
 // LoadAndRun will load configurations, initialize components and then start the related process to serve requests.
 // Return error if meet any problems.
-func (bs *Bootstrap) LoadAndRun() {
+func (bs *Bootstrap) LoadAndRun(ctx context.Context, cancel context.CancelFunc) {
-	// Create the root context
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
 	rootContext := &env.Context{
 		SystemContext: ctx,
 		WG:            &sync.WaitGroup{},
@@ -110,10 +106,6 @@ func (bs *Bootstrap) LoadAndRun() {
 	apiServer := bs.loadAndRunAPIServer(rootContext, config.DefaultConfig, ctl)
 	logger.Infof("Server is started at %s:%d with %s", "", config.DefaultConfig.Port, config.DefaultConfig.Protocol)
 
-	// Start outdated log files sweeper
-	logSweeper := logger.NewSweeper(ctx, config.GetLogBasePath(), config.GetLogArchivePeriod())
-	logSweeper.Start()
-
 	// To indicate if any errors occurred
 	var err error
 	// Block here