Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2025-12-05 04:54:30 +01:00
Code Issues Projects Releases Wiki Activity

Allow disable audit log to DB when initialize (#22452)

Browse Source 
Fixes #22257

Add support for SKIP_LOG_AUDIT_DATABASE env to control audit log database storage during Harbor initialization.

Signed-off-by: wang yan <yan-yw.wang@broadcom.com>
Co-authored-by: wang yan <yan-yw.wang@broadcom.com>
This commit is contained in:
Wang Yan 2025-10-17 12:01:46 +08:00 committed by GitHub
parent e7066476a3
commit 6e15beb160
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 501 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
src/core/main.go
View File

@ -22,12 +22,14 @@ import (
"net/url"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"github.com/beego/beego/v2/server/web"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
common_http "github.com/goharbor/harbor/src/common/http"
configCtl "github.com/goharbor/harbor/src/controller/config"
@ -222,6 +224,10 @@ func main() {
log.Error(err)
}
// Allow user to disable writing audit log to db by env while initialize
if err := initSkipAuditDBbyEnv(ctx); err != nil {
log.Errorf("Failed to initialize SkipAuditDB by ENV: %v", err)
}
// Init API handler
if err := api.Init(); err != nil {
log.Fatalf("Failed to initialize API handlers with error: %s", err.Error())
@ -356,3 +362,34 @@ func getDefaultScannerName() string {
}
return ""
}
func initSkipAuditDBbyEnv(ctx context.Context) error {
var err error
skipAuditEnv := false
s := os.Getenv("SKIP_LOG_AUDIT_DATABASE")
if s != "" {
skipAuditEnv, err = strconv.ParseBool(s)
if err != nil {
log.Warningf("Failed to parse SKIP_LOG_AUDIT_DATABASE to bool with error: %v, Will use SKIP_LOG_AUDIT_DATABASE env as false", err)
}
}
log.Debugf("get SKIP_LOG_AUDIT_DATABASE from Env is %v", skipAuditEnv)
// get from db
mgr := config.GetCfgManager(ctx)
cfg, err := mgr.GetItemFromDriver(ctx, common.SkipAuditLogDatabase)
if err != nil {
return err
}
// if key not exist in the db, set default ENV value
if val, ok := cfg[common.SkipAuditLogDatabase]; !ok {
log.Debugf("key SkipAuditLogDatabase do not exist in the db, will initialize as %v", skipAuditEnv)
cfg[common.SkipAuditLogDatabase] = skipAuditEnv
if err := mgr.UpdateConfig(ctx, cfg); err != nil {
return err
}
} else {
log.Debugf("key SkipAuditLogDatabase aleady exist in the db with value %v", val)
}
return nil
}

1
src/lib/config/config.go
View File

@ -48,6 +48,7 @@ type Manager interface {
Set(ctx context.Context, key string, value any)
Save(ctx context.Context) error
Get(ctx context.Context, key string) *metadata.ConfigureValue
GetItemFromDriver(ctx context.Context, key string) (map[string]any, error)
UpdateConfig(ctx context.Context, cfgs map[string]any) error
GetUserCfgs(ctx context.Context) map[string]any
ValidateCfg(ctx context.Context, cfgs map[string]any) error

5
src/pkg/config/db/cache.go
View File

@ -61,6 +61,11 @@ func (d *Cache) Save(ctx context.Context, cfg map[string]any) error {
return nil
}
// Get - delegate to driver
func (d *Cache) Get(ctx context.Context, key string) (map[string]any, error) {
return d.driver.Get(ctx, key)
}
// NewCacheDriver returns driver with cache
func NewCacheDriver(cache cache.Cache, driver store.Driver) store.Driver {
return &Cache{

17
src/pkg/config/db/dao/dao.go
View File

@ -22,6 +22,7 @@ import (
"github.com/goharbor/harbor/src/lib/config/models"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
)
// DAO the dao for configure items
@ -30,6 +31,8 @@ type DAO interface {
GetConfigEntries(ctx context.Context) ([]*models.ConfigEntry, error)
// SaveConfigEntries save configure items provided
SaveConfigEntries(ctx context.Context, entries []models.ConfigEntry) error
// GetConfigItem get configure item by key
GetConfigItem(ctx context.Context, query *q.Query) ([]*models.ConfigEntry, error)
}
type dao struct {
@ -85,3 +88,17 @@ func (d *dao) SaveConfigEntries(ctx context.Context, entries []models.ConfigEntr
}
return nil
}
// GetConfigItem get configure item by query
func (d *dao) GetConfigItem(ctx context.Context, query *q.Query) ([]*models.ConfigEntry, error) {
query = q.MustClone(query)
qs, err := orm.QuerySetter(ctx, &models.ConfigEntry{}, query)
if err != nil {
return nil, err
}
var configs []*models.ConfigEntry
if _, err := qs.All(&configs); err != nil {
return nil, err
}
return configs, nil
}

16
src/pkg/config/db/db.go
View File

@ -23,6 +23,7 @@ import (
"github.com/goharbor/harbor/src/lib/config/models"
"github.com/goharbor/harbor/src/lib/encrypt"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/config/db/dao"
)
@ -84,3 +85,18 @@ func (d *Database) Save(ctx context.Context, cfgs map[string]any) error {
}
return d.cfgDAO.SaveConfigEntries(ctx, configEntries)
}
// Get - Get config item from db
func (d *Database) Get(ctx context.Context, key string) (map[string]any, error) {
resultMap := map[string]any{}
configEntries, err := d.cfgDAO.GetConfigItem(ctx, q.New(q.KeyWords{"k": key}))
if err != nil {
log.Debugf("get config db error: %v", err)
return resultMap, err
}
// convert to map if there's any record
for _, item := range configEntries {
resultMap[item.Key] = item.Value
}
return resultMap, nil
}

6
src/pkg/config/inmemory/manager.go
View File

@ -16,6 +16,7 @@ package inmemory
import (
"context"
"errors"
"maps"
"sync"
@ -54,6 +55,11 @@ func (d *Driver) Save(_ context.Context, cfg map[string]any) error {
return nil
}
// TODO
func (d *Driver) Get(_ context.Context, _ string) (map[string]any, error) {
return nil, errors.ErrUnsupported
}
// NewInMemoryManager create a manager for unit testing, doesn't involve database or REST
func NewInMemoryManager() *config.CfgManager {
manager := &config.CfgManager{Store: store.NewConfigStore(&Driver{cfgMap: map[string]any{}})}

5
src/pkg/config/manager.go
View File

@ -180,6 +180,11 @@ func (c *CfgManager) UpdateConfig(ctx context.Context, cfgs map[string]any) erro
return c.Store.Update(ctx, cfgs)
}
// GetItemFromDriver ...
func (c *CfgManager) GetItemFromDriver(ctx context.Context, key string) (map[string]any, error) {
return c.Store.GetFromDriver(ctx, key)
}
// ValidateCfg validate config by metadata. return the first error if exist.
func (c *CfgManager) ValidateCfg(ctx context.Context, cfgs map[string]any) error {
for key, value := range cfgs {

153
src/pkg/config/manager_test.go Normal file
View File

@ -0,0 +1,153 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"context"
"testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/config/store"
)
// MockDriver is a mock implementation of store.Driver
type MockDriver struct {
mock.Mock
}
func (m *MockDriver) Load(ctx context.Context) (map[string]any, error) {
args := m.Called(ctx)
return args.Get(0).(map[string]any), args.Error(1)
}
func (m *MockDriver) Save(ctx context.Context, cfg map[string]any) error {
args := m.Called(ctx, cfg)
return args.Error(0)
}
func (m *MockDriver) Get(ctx context.Context, key string) (map[string]any, error) {
args := m.Called(ctx, key)
return args.Get(0).(map[string]any), args.Error(1)
}
// GetItemFromDriverTestSuite tests the GetItemFromDriver method
type GetItemFromDriverTestSuite struct {
suite.Suite
ctx context.Context
manager *CfgManager
driver *MockDriver
}
func (suite *GetItemFromDriverTestSuite) SetupTest() {
suite.ctx = context.Background()
suite.driver = &MockDriver{}
suite.manager = &CfgManager{
Store: store.NewConfigStore(suite.driver),
}
}
func (suite *GetItemFromDriverTestSuite) TestGetItemFromDriverSuccess() {
key := common.SkipAuditLogDatabase
expectedResult := map[string]any{
common.SkipAuditLogDatabase: true,
}
suite.driver.On("Get", suite.ctx, key).Return(expectedResult, nil)
result, err := suite.manager.GetItemFromDriver(suite.ctx, key)
suite.Require().NoError(err)
suite.Equal(expectedResult, result)
suite.driver.AssertExpectations(suite.T())
}
func (suite *GetItemFromDriverTestSuite) TestGetItemFromDriverError() {
key := common.SkipAuditLogDatabase
expectedError := errors.New("database connection failed")
suite.driver.On("Get", suite.ctx, key).Return(map[string]any{}, expectedError)
result, err := suite.manager.GetItemFromDriver(suite.ctx, key)
suite.Require().Error(err)
suite.Equal(expectedError, err)
suite.Empty(result)
suite.driver.AssertExpectations(suite.T())
}
func (suite *GetItemFromDriverTestSuite) TestGetItemFromDriverEmptyResult() {
key := common.SkipAuditLogDatabase
expectedResult := map[string]any{}
suite.driver.On("Get", suite.ctx, key).Return(expectedResult, nil)
result, err := suite.manager.GetItemFromDriver(suite.ctx, key)
suite.Require().NoError(err)
suite.Equal(expectedResult, result)
suite.driver.AssertExpectations(suite.T())
}
func (suite *GetItemFromDriverTestSuite) TestGetItemFromDriverMultipleKeys() {
key := common.AuditLogForwardEndpoint
expectedResult := map[string]any{
common.AuditLogForwardEndpoint: "syslog://localhost:514",
common.SkipAuditLogDatabase: false,
}
suite.driver.On("Get", suite.ctx, key).Return(expectedResult, nil)
result, err := suite.manager.GetItemFromDriver(suite.ctx, key)
suite.Require().NoError(err)
suite.Equal(expectedResult, result)
suite.driver.AssertExpectations(suite.T())
}
func (suite *GetItemFromDriverTestSuite) TestGetItemFromDriverNilContext() {
key := common.SkipAuditLogDatabase
expectedResult := map[string]any{
common.SkipAuditLogDatabase: false,
}
suite.driver.On("Get", mock.Anything, key).Return(expectedResult, nil)
result, err := suite.manager.GetItemFromDriver(nil, key)
suite.Require().NoError(err)
suite.Equal(expectedResult, result)
suite.driver.AssertExpectations(suite.T())
}
func (suite *GetItemFromDriverTestSuite) TestGetItemFromDriverEmptyKey() {
key := ""
expectedResult := map[string]any{}
suite.driver.On("Get", suite.ctx, key).Return(expectedResult, nil)
result, err := suite.manager.GetItemFromDriver(suite.ctx, key)
suite.Require().NoError(err)
suite.Equal(expectedResult, result)
suite.driver.AssertExpectations(suite.T())
}
func TestGetItemFromDriverTestSuite(t *testing.T) {
suite.Run(t, new(GetItemFromDriverTestSuite))
}

5
src/pkg/config/rest/rest.go
View File

@ -62,3 +62,8 @@ func (h *Driver) Load(_ context.Context) (map[string]any, error) {
func (h *Driver) Save(_ context.Context, cfg map[string]any) error {
return h.client.Put(h.configRESTURL, cfg)
}
// TODO
func (h *Driver) Get(_ context.Context, _ string) (map[string]any, error) {
return nil, errors.ErrUnsupported
}

2
src/pkg/config/store/driver.go
View File

@ -23,4 +23,6 @@ type Driver interface {
Load(ctx context.Context) (map[string]any, error)
// Save - save config item into config driver
Save(ctx context.Context, cfg map[string]any) error
// Get - get config item from config driver
Get(ctx context.Context, key string) (map[string]any, error)
}

12
src/pkg/config/store/store.go
View File

@ -51,6 +51,18 @@ func (c *ConfigStore) Get(key string) (*metadata.ConfigureValue, error) {
return nil, metadata.ErrValueNotSet
}
// GetFromDriver ...
func (c *ConfigStore) GetFromDriver(ctx context.Context, key string) (map[string]any, error) {
if c.cfgDriver == nil {
return nil, errors.New("failed to load store, cfgDriver is nil")
}
cfgs, err := c.cfgDriver.Get(ctx, key)
if err != nil {
return nil, err
}
return cfgs, nil
}
// GetAnyType get any type for config items
func (c *ConfigStore) GetAnyType(key string) (any, error) {
if value, ok := c.cfgValues.Load(key); ok {

212
src/pkg/config/store/store_test.go Normal file
View File

@ -0,0 +1,212 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package store
import (
"context"
"testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/lib/errors"
)
// MockDriver is a mock implementation of store.Driver
type MockDriver struct {
mock.Mock
}
func (m *MockDriver) Load(ctx context.Context) (map[string]any, error) {
args := m.Called(ctx)
return args.Get(0).(map[string]any), args.Error(1)
}
func (m *MockDriver) Save(ctx context.Context, cfg map[string]any) error {
args := m.Called(ctx, cfg)
return args.Error(0)
}
func (m *MockDriver) Get(ctx context.Context, key string) (map[string]any, error) {
args := m.Called(ctx, key)
return args.Get(0).(map[string]any), args.Error(1)
}
// GetFromDriverTestSuite tests the GetFromDriver method in ConfigStore
type GetFromDriverTestSuite struct {
suite.Suite
ctx context.Context
store *ConfigStore
driver *MockDriver
}
func (suite *GetFromDriverTestSuite) SetupTest() {
suite.ctx = context.Background()
suite.driver = &MockDriver{}
suite.store = &ConfigStore{
cfgDriver: suite.driver,
}
}
func (suite *GetFromDriverTestSuite) TestGetFromDriverSuccess() {
key := common.SkipAuditLogDatabase
expectedResult := map[string]any{
common.SkipAuditLogDatabase: true,
}
suite.driver.On("Get", suite.ctx, key).Return(expectedResult, nil)
result, err := suite.store.GetFromDriver(suite.ctx, key)
suite.Require().NoError(err)
suite.Equal(expectedResult, result)
suite.driver.AssertExpectations(suite.T())
}
func (suite *GetFromDriverTestSuite) TestGetFromDriverNilDriver() {
key := common.SkipAuditLogDatabase
suite.store.cfgDriver = nil
result, err := suite.store.GetFromDriver(suite.ctx, key)
suite.Require().Error(err)
suite.Contains(err.Error(), "failed to load store, cfgDriver is nil")
suite.Nil(result)
}
func (suite *GetFromDriverTestSuite) TestGetFromDriverError() {
key := common.SkipAuditLogDatabase
expectedError := errors.New("database connection failed")
suite.driver.On("Get", suite.ctx, key).Return(map[string]any{}, expectedError)
result, err := suite.store.GetFromDriver(suite.ctx, key)
suite.Require().Error(err)
suite.Equal(expectedError, err)
suite.Empty(result)
suite.driver.AssertExpectations(suite.T())
}
func (suite *GetFromDriverTestSuite) TestGetFromDriverEmptyResult() {
key := common.SkipAuditLogDatabase
expectedResult := map[string]any{}
suite.driver.On("Get", suite.ctx, key).Return(expectedResult, nil)
result, err := suite.store.GetFromDriver(suite.ctx, key)
suite.Require().NoError(err)
suite.Equal(expectedResult, result)
suite.driver.AssertExpectations(suite.T())
}
func (suite *GetFromDriverTestSuite) TestGetFromDriverMultipleConfigs() {
key := common.AuditLogForwardEndpoint
expectedResult := map[string]any{
common.AuditLogForwardEndpoint: "syslog://localhost:514",
common.SkipAuditLogDatabase: false,
"other_config": "value",
}
suite.driver.On("Get", suite.ctx, key).Return(expectedResult, nil)
result, err := suite.store.GetFromDriver(suite.ctx, key)
suite.Require().NoError(err)
suite.Equal(expectedResult, result)
suite.Equal("syslog://localhost:514", result[common.AuditLogForwardEndpoint])
suite.Equal(false, result[common.SkipAuditLogDatabase])
suite.Equal("value", result["other_config"])
suite.driver.AssertExpectations(suite.T())
}
func (suite *GetFromDriverTestSuite) TestGetFromDriverNilContext() {
key := common.SkipAuditLogDatabase
expectedResult := map[string]any{
common.SkipAuditLogDatabase: false,
}
suite.driver.On("Get", mock.Anything, key).Return(expectedResult, nil)
result, err := suite.store.GetFromDriver(nil, key)
suite.Require().NoError(err)
suite.Equal(expectedResult, result)
suite.driver.AssertExpectations(suite.T())
}
func (suite *GetFromDriverTestSuite) TestGetFromDriverEmptyKey() {
key := ""
expectedResult := map[string]any{}
suite.driver.On("Get", suite.ctx, key).Return(expectedResult, nil)
result, err := suite.store.GetFromDriver(suite.ctx, key)
suite.Require().NoError(err)
suite.Equal(expectedResult, result)
suite.driver.AssertExpectations(suite.T())
}
func (suite *GetFromDriverTestSuite) TestGetFromDriverDifferentKeys() {
testCases := []struct {
name string
key string
expectedResult map[string]any
}{
{
name: "skip_audit_log_database",
key: common.SkipAuditLogDatabase,
expectedResult: map[string]any{
common.SkipAuditLogDatabase: true,
},
},
{
name: "audit_log_forward_endpoint",
key: common.AuditLogForwardEndpoint,
expectedResult: map[string]any{
common.AuditLogForwardEndpoint: "syslog://remote:514",
},
},
{
name: "pull_audit_log_disable",
key: common.PullAuditLogDisable,
expectedResult: map[string]any{
common.PullAuditLogDisable: false,
},
},
}
for _, tc := range testCases {
suite.Run(tc.name, func() {
suite.driver.On("Get", suite.ctx, tc.key).Return(tc.expectedResult, nil)
result, err := suite.store.GetFromDriver(suite.ctx, tc.key)
suite.Require().NoError(err)
suite.Equal(tc.expectedResult, result)
suite.driver.AssertExpectations(suite.T())
// Reset mock for next iteration
suite.driver.ExpectedCalls = nil
})
}
}
func TestGetFromDriverTestSuite(t *testing.T) {
suite.Run(t, new(GetFromDriverTestSuite))
}

30
src/testing/lib/config/manager.go
View File

@ -76,6 +76,36 @@ func (_m *Manager) GetDatabaseCfg() *models.Database {
return r0
}
// GetItemFromDriver provides a mock function with given fields: ctx, key
func (_m *Manager) GetItemFromDriver(ctx context.Context, key string) (map[string]interface{}, error) {
ret := _m.Called(ctx, key)
if len(ret) == 0 {
panic("no return value specified for GetItemFromDriver")
}
var r0 map[string]interface{}
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) (map[string]interface{}, error)); ok {
return rf(ctx, key)
}
if rf, ok := ret.Get(0).(func(context.Context, string) map[string]interface{}); ok {
r0 = rf(ctx, key)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(map[string]interface{})
}
}
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, key)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetUserCfgs provides a mock function with given fields: ctx
func (_m *Manager) GetUserCfgs(ctx context.Context) map[string]interface{} {
ret := _m.Called(ctx)