Merge pull request #16833 from stonezdj/22may10_purge_audit_log

(feat) Add job service to purge audit_log
This commit is contained in:
stonezdj(Daojun Zhang) 2022-05-17 09:02:56 +08:00 committed by GitHub
commit 2852a7606b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 1096 additions and 44 deletions

View File

@ -205,4 +205,8 @@ const (
// DefaultCacheExpireHours is the default cache expire hours, default is
// 24h.
DefaultCacheExpireHours = 24
PurgeAuditIncludeOperations = "include_operations"
PurgeAuditDryRun = "dry_run"
PurgeAuditRetentionHour = "audit_retention_hour"
)

View File

@ -75,4 +75,5 @@ const (
ResourceReplicationPolicy = Resource("replication-policy")
ResourceScanAll = Resource("scan-all")
ResourceSystemVolumes = Resource("system-volumes")
ResourcePurgeAuditLog = Resource("purge-audit")
)

View File

@ -16,6 +16,7 @@ package event
import (
"fmt"
"github.com/goharbor/harbor/src/common/rbac"
proModels "github.com/goharbor/harbor/src/pkg/project/models"
"time"
@ -65,7 +66,7 @@ func (c *CreateProjectEvent) ResolveToAuditLog() (*model.AuditLog, error) {
auditLog := &model.AuditLog{
ProjectID: c.ProjectID,
OpTime: c.OccurAt,
Operation: "create",
Operation: rbac.ActionCreate.String(),
Username: c.Operator,
ResourceType: "project",
Resource: c.Project}
@ -91,7 +92,7 @@ func (d *DeleteProjectEvent) ResolveToAuditLog() (*model.AuditLog, error) {
auditLog := &model.AuditLog{
ProjectID: d.ProjectID,
OpTime: d.OccurAt,
Operation: "delete",
Operation: rbac.ActionDelete.String(),
Username: d.Operator,
ResourceType: "project",
Resource: d.Project}
@ -117,7 +118,7 @@ func (d *DeleteRepositoryEvent) ResolveToAuditLog() (*model.AuditLog, error) {
auditLog := &model.AuditLog{
ProjectID: d.ProjectID,
OpTime: d.OccurAt,
Operation: "delete",
Operation: rbac.ActionDelete.String(),
Username: d.Operator,
ResourceType: "repository",
Resource: d.Repository,
@ -156,7 +157,7 @@ func (p *PushArtifactEvent) ResolveToAuditLog() (*model.AuditLog, error) {
auditLog := &model.AuditLog{
ProjectID: p.Artifact.ProjectID,
OpTime: p.OccurAt,
Operation: "create",
Operation: rbac.ActionCreate.String(),
Username: p.Operator,
ResourceType: "artifact"}
@ -185,7 +186,7 @@ func (p *PullArtifactEvent) ResolveToAuditLog() (*model.AuditLog, error) {
auditLog := &model.AuditLog{
ProjectID: p.Artifact.ProjectID,
OpTime: p.OccurAt,
Operation: "pull",
Operation: rbac.ActionPull.String(),
Username: p.Operator,
ResourceType: "artifact"}
@ -221,7 +222,7 @@ func (d *DeleteArtifactEvent) ResolveToAuditLog() (*model.AuditLog, error) {
auditLog := &model.AuditLog{
ProjectID: d.Artifact.ProjectID,
OpTime: d.OccurAt,
Operation: "delete",
Operation: rbac.ActionDelete.String(),
Username: d.Operator,
ResourceType: "artifact",
Resource: fmt.Sprintf("%s:%s", d.Artifact.RepositoryName, d.Artifact.Digest)}
@ -247,7 +248,7 @@ func (c *CreateTagEvent) ResolveToAuditLog() (*model.AuditLog, error) {
auditLog := &model.AuditLog{
ProjectID: c.AttachedArtifact.ProjectID,
OpTime: c.OccurAt,
Operation: "create",
Operation: rbac.ActionCreate.String(),
Username: c.Operator,
ResourceType: "tag",
Resource: fmt.Sprintf("%s:%s", c.Repository, c.Tag)}
@ -275,7 +276,7 @@ func (d *DeleteTagEvent) ResolveToAuditLog() (*model.AuditLog, error) {
auditLog := &model.AuditLog{
ProjectID: d.AttachedArtifact.ProjectID,
OpTime: d.OccurAt,
Operation: "delete",
Operation: rbac.ActionDelete.String(),
Username: d.Operator,
ResourceType: "tag",
Resource: fmt.Sprintf("%s:%s", d.Repository, d.Tag)}

View File

@ -0,0 +1,68 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package jobservice
import (
"context"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/scheduler"
)
var (
// SchedulerCtl ...
SchedulerCtl = NewSchedulerCtrl()
)
// SchedulerController interface to manage schedule
type SchedulerController interface {
// Get the schedule
Get(ctx context.Context, vendorType string) (*scheduler.Schedule, error)
// Create with cron type & string
Create(ctx context.Context, vendorType, cronType, cron, callbackFuncName string, policy interface{}, extrasParam map[string]interface{}) (int64, error)
// Delete the schedule
Delete(ctx context.Context, vendorType string) error
}
type schedulerController struct {
schedulerMgr scheduler.Scheduler
}
// NewSchedulerCtrl ...
func NewSchedulerCtrl() SchedulerController {
return &schedulerController{schedulerMgr: scheduler.New()}
}
func (s schedulerController) Get(ctx context.Context, vendorType string) (*scheduler.Schedule, error) {
sch, err := s.schedulerMgr.ListSchedules(ctx, q.New(q.KeyWords{"VendorType": vendorType}))
if err != nil {
return nil, err
}
if len(sch) == 0 {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no schedule is found")
}
if sch[0] == nil {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).WithMessage("no schedule is found")
}
return sch[0], nil
}
func (s schedulerController) Create(ctx context.Context, vendorType, cronType, cron, callbackFuncName string,
policy interface{}, extrasParam map[string]interface{}) (int64, error) {
return s.schedulerMgr.Schedule(ctx, vendorType, -1, cronType, cron, callbackFuncName, policy, extrasParam)
}
func (s schedulerController) Delete(ctx context.Context, vendorType string) error {
return s.schedulerMgr.UnScheduleByVendor(ctx, vendorType, -1)
}

View File

@ -0,0 +1,70 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package jobservice
import (
"github.com/goharbor/harbor/src/controller/purge"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/testing/mock"
testingScheduler "github.com/goharbor/harbor/src/testing/pkg/scheduler"
"github.com/stretchr/testify/suite"
"testing"
)
type ScheduleTestSuite struct {
suite.Suite
scheduler *testingScheduler.Scheduler
ctl SchedulerController
}
func (s *ScheduleTestSuite) SetupSuite() {
s.scheduler = &testingScheduler.Scheduler{}
s.ctl = &schedulerController{
schedulerMgr: s.scheduler,
}
}
func (s *ScheduleTestSuite) TestCreateSchedule() {
s.scheduler.On("Schedule", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
dataMap := make(map[string]interface{})
p := purge.JobPolicy{}
id, err := s.ctl.Create(nil, purge.VendorType, "Daily", "* * * * * *", purge.SchedulerCallback, p, dataMap)
s.Nil(err)
s.Equal(int64(1), id)
}
func (s *ScheduleTestSuite) TestDeleteSchedule() {
s.scheduler.On("UnScheduleByVendor", mock.Anything, mock.Anything, mock.Anything).Return(nil)
s.Nil(s.ctl.Delete(nil, purge.VendorType))
}
func (s *ScheduleTestSuite) TestGetSchedule() {
s.scheduler.On("ListSchedules", mock.Anything, mock.Anything).Return([]*scheduler.Schedule{
{
ID: 1,
VendorType: purge.VendorType,
},
}, nil)
schedule, err := s.ctl.Get(nil, purge.VendorType)
s.Nil(err)
s.Equal(purge.VendorType, schedule.VendorType)
}
func TestScheduleTestSuite(t *testing.T) {
suite.Run(t, &ScheduleTestSuite{})
}

View File

@ -0,0 +1,95 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package purge
import (
"context"
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/scheduler"
"github.com/goharbor/harbor/src/pkg/task"
)
const (
// SchedulerCallback ...
SchedulerCallback = "PURGE_AUDIT_LOG_CALLBACK"
// VendorType ...
VendorType = "PURGE_AUDIT_LOG"
)
// Ctrl a global purge controller instance
var Ctrl = NewController()
func init() {
err := scheduler.RegisterCallbackFunc(SchedulerCallback, purgeCallback)
if err != nil {
log.Fatalf("failed to registry purge job call back, %v", err)
}
}
func purgeCallback(ctx context.Context, p string) error {
param := &JobPolicy{}
if err := json.Unmarshal([]byte(p), param); err != nil {
return fmt.Errorf("failed to unmashal the param: %v", err)
}
_, err := Ctrl.Start(ctx, *param, task.ExecutionTriggerSchedule)
return err
}
// Controller defines the interface with the purge job
type Controller interface {
// Start kick off a purge schedule
Start(ctx context.Context, policy JobPolicy, trigger string) (int64, error)
}
type controller struct {
taskMgr task.Manager
exeMgr task.ExecutionManager
}
func (c *controller) Start(ctx context.Context, policy JobPolicy, trigger string) (int64, error) {
para := make(map[string]interface{})
para[common.PurgeAuditDryRun] = policy.DryRun
para[common.PurgeAuditRetentionHour] = policy.RetentionHour
para[common.PurgeAuditIncludeOperations] = policy.IncludeOperations
execID, err := c.exeMgr.Create(ctx, VendorType, -1, trigger, para)
if err != nil {
return -1, err
}
_, err = c.taskMgr.Create(ctx, execID, &task.Job{
Name: job.PurgeAudit,
Metadata: &job.Metadata{
JobKind: job.KindGeneric,
},
Parameters: para,
})
if err != nil {
return -1, err
}
return execID, nil
}
// NewController ...
func NewController() Controller {
return &controller{
taskMgr: task.NewManager(),
exeMgr: task.NewExecutionManager(),
}
}

View File

@ -0,0 +1,55 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package purge
import (
"github.com/goharbor/harbor/src/pkg/task"
testingTask "github.com/goharbor/harbor/src/testing/pkg/task"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"testing"
)
type PurgeControllerTestSuite struct {
suite.Suite
taskMgr *testingTask.Manager
exeMgr *testingTask.ExecutionManager
Ctl Controller
}
func (p *PurgeControllerTestSuite) SetupSuite() {
p.taskMgr = &testingTask.Manager{}
p.exeMgr = &testingTask.ExecutionManager{}
p.Ctl = &controller{
taskMgr: p.taskMgr,
exeMgr: p.exeMgr,
}
}
func (p *PurgeControllerTestSuite) TestStart() {
p.exeMgr.On("Create", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
p.taskMgr.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil)
policy := JobPolicy{}
id, err := p.Ctl.Start(nil, policy, task.ExecutionTriggerManual)
p.Nil(err)
p.Equal(int64(1), id)
}
func (p *PurgeControllerTestSuite) TearDownSuite() {
}
func TestPurgeControllerTestSuite(t *testing.T) {
suite.Run(t, &PurgeControllerTestSuite{})
}

View File

@ -0,0 +1,52 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package purge
import (
"encoding/json"
"github.com/goharbor/harbor/src/lib/log"
)
// JobPolicy defines the purge job policy
type JobPolicy struct {
Trigger *Trigger `json:"trigger"`
DryRun bool `json:"dryrun"`
RetentionHour int `json:"retention_hour"`
IncludeOperations string `json:"include_operations"`
ExtraAttrs map[string]interface{} `json:"extra_attrs"`
}
// TriggerType represents the type of trigger.
type TriggerType string
// Trigger holds info for a trigger
type Trigger struct {
Type TriggerType `json:"type"`
Settings *TriggerSettings `json:"trigger_settings"`
}
// TriggerSettings is the setting about the trigger
type TriggerSettings struct {
Cron string `json:"cron"`
}
// String convert map to json string
func String(extras map[string]interface{}) string {
result, err := json.Marshal(extras)
if err != nil {
log.Errorf("failed to convert to json string, value %+v", extras)
}
return string(result)
}

View File

@ -0,0 +1,38 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package purge
import "testing"
func TestString(t *testing.T) {
type args struct {
extras map[string]interface{}
}
tests := []struct {
name string
args args
want string
}{
{"normal", args{map[string]interface{}{"dry_run": true, "audit_log_retention_hour": 168}}, "{\"audit_log_retention_hour\":168,\"dry_run\":true}"},
{"empty", args{map[string]interface{}{}}, "{}"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := String(tt.args.extras); got != tt.want {
t.Errorf("String() = %v, want %v", got, tt.want)
}
})
}
}

View File

@ -0,0 +1,117 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package purge
import (
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/audit"
"os"
"strings"
)
// Job defines the purge job
type Job struct {
retentionHour int
includeOperations []string
dryRun bool
auditMgr audit.Manager
}
// MaxFails is implementation of same method in Interface.
func (j *Job) MaxFails() uint {
return 1
}
// MaxCurrency is implementation of same method in Interface.
func (j *Job) MaxCurrency() uint {
return 1
}
// ShouldRetry ...
func (j *Job) ShouldRetry() bool {
return false
}
// Validate is implementation of same method in Interface.
func (j *Job) Validate(params job.Parameters) error {
return nil
}
func (j *Job) parseParams(params job.Parameters) {
if params == nil || len(params) == 0 {
return
}
retHr, exist := params[common.PurgeAuditRetentionHour]
if !exist {
return
}
if rh, ok := retHr.(int); ok {
j.retentionHour = rh
} else if rh, ok := retHr.(float64); ok {
j.retentionHour = int(rh)
}
dryRun, exist := params[common.PurgeAuditDryRun]
if exist {
if dryRun, ok := dryRun.(bool); ok && dryRun {
j.dryRun = dryRun
}
}
j.includeOperations = []string{}
operations, exist := params[common.PurgeAuditIncludeOperations]
if exist {
if includeOps, ok := operations.(string); ok {
if len(includeOps) > 0 {
j.includeOperations = strings.Split(includeOps, ",")
}
}
}
// UT will use the mock mgr
if os.Getenv("UTTEST") != "true" {
j.auditMgr = audit.Mgr
}
}
// Run the purge logic here.
func (j *Job) Run(ctx job.Context, params job.Parameters) error {
logger := ctx.GetLogger()
logger.Info("Purge audit job start")
logger.Infof("job parameters %+v", params)
j.parseParams(params)
ormCtx := ctx.SystemContext()
if j.retentionHour == -1 || j.retentionHour == 0 {
log.Infof("quit purge job, retentionHour:%v ", j.retentionHour)
return nil
}
n, err := j.auditMgr.Purge(ormCtx, j.retentionHour, j.includeOperations, j.dryRun)
if err != nil {
logger.Errorf("failed to purge audit log, error: %v", err)
return err
}
logger.Infof("Purge operation parameter, renention_hour=%v, include_operations:%v, dry_run:%v",
j.retentionHour, j.includeOperations, j.dryRun)
if j.dryRun {
logger.Infof("[DRYRUN]Purged %d rows of audit logs", n)
} else {
logger.Infof("Purged %d rows of audit logs", n)
}
// Successfully exit
return nil
}

View File

@ -0,0 +1,82 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package purge
import (
"fmt"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/audit"
htesting "github.com/goharbor/harbor/src/testing"
mockjobservice "github.com/goharbor/harbor/src/testing/jobservice"
"github.com/goharbor/harbor/src/testing/mock"
mockAudit "github.com/goharbor/harbor/src/testing/pkg/audit"
"github.com/stretchr/testify/suite"
"testing"
)
type PurgeJobTestSuite struct {
htesting.Suite
auditMgr audit.Manager
}
func (suite *PurgeJobTestSuite) SetupSuite() {
suite.auditMgr = &mockAudit.Manager{}
}
func (suite *PurgeJobTestSuite) TearDownSuite() {
}
func (suite *PurgeJobTestSuite) TestParseParams() {
ctx := &mockjobservice.MockJobContext{}
logger := &mockjobservice.MockJobLogger{}
ctx.On("GetLogger").Return(logger)
j := &Job{}
param := job.Parameters{common.PurgeAuditRetentionHour: 128, common.PurgeAuditDryRun: true}
j.parseParams(param)
suite.Require().Equal(true, j.dryRun)
suite.Require().Equal(128, j.retentionHour)
suite.Require().Equal([]string{}, j.includeOperations)
j2 := &Job{}
param2 := job.Parameters{common.PurgeAuditRetentionHour: 24, common.PurgeAuditDryRun: false, common.PurgeAuditIncludeOperations: "Delete,Create,Pull"}
j2.parseParams(param2)
suite.Require().Equal(false, j2.dryRun)
suite.Require().Equal(24, j2.retentionHour)
suite.Require().Equal([]string{"Delete", "Create", "Pull"}, j2.includeOperations)
}
func (suite *PurgeJobTestSuite) TestRun() {
ctx := &mockjobservice.MockJobContext{}
logger := &mockjobservice.MockJobLogger{}
ctx.On("GetLogger").Return(logger)
auditManager := &mockAudit.Manager{}
auditManager.On("Purge", mock.Anything, 128, []string{}, true).Return(int64(100), nil)
j := &Job{auditMgr: auditManager}
param := job.Parameters{common.PurgeAuditRetentionHour: 128, common.PurgeAuditDryRun: true}
ret := j.Run(ctx, param)
suite.Require().Nil(ret)
auditManager.On("Purge", mock.Anything, 24, []string{}, false).Return(int64(0), fmt.Errorf("failed to connect database"))
j2 := &Job{auditMgr: auditManager}
param2 := job.Parameters{common.PurgeAuditRetentionHour: 24, common.PurgeAuditDryRun: false}
ret2 := j2.Run(ctx, param2)
suite.Require().NotNil(ret2)
}
func TestPurgeJobTestSuite(t *testing.T) {
suite.Run(t, &PurgeJobTestSuite{})
}

View File

@ -34,4 +34,6 @@ const (
Retention = "RETENTION"
// P2PPreheat : the name of the P2P preheat job
P2PPreheat = "P2P_PREHEAT"
// PurgeAudit : the name of purge audit job
PurgeAudit = "PURGE_AUDIT"
)

View File

@ -35,6 +35,7 @@ import (
"github.com/goharbor/harbor/src/jobservice/job/impl/gc"
"github.com/goharbor/harbor/src/jobservice/job/impl/legacy"
"github.com/goharbor/harbor/src/jobservice/job/impl/notification"
"github.com/goharbor/harbor/src/jobservice/job/impl/purge"
"github.com/goharbor/harbor/src/jobservice/job/impl/replication"
"github.com/goharbor/harbor/src/jobservice/job/impl/sample"
"github.com/goharbor/harbor/src/jobservice/lcm"
@ -308,6 +309,7 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool(
// Functional jobs
job.ImageScanJob: (*scan.Job)(nil),
job.GarbageCollection: (*gc.GarbageCollector)(nil),
job.PurgeAudit: (*purge.Job)(nil),
job.Replication: (*replication.Replication)(nil),
job.Retention: (*retention.Job)(nil),
scheduler.JobNameScheduler: (*scheduler.PeriodicJob)(nil),

View File

@ -16,7 +16,13 @@ package dao
import (
"context"
"strings"
beegorm "github.com/beego/beego/orm"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/audit/model"
@ -34,6 +40,8 @@ type DAO interface {
Get(ctx context.Context, id int64) (access *model.AuditLog, err error)
// Delete the audit log specified by ID
Delete(ctx context.Context, id int64) (err error)
// Purge the audit log
Purge(ctx context.Context, retentionHour int, includeOperations []string, dryRun bool) (int64, error)
}
// New returns an instance of the default DAO
@ -41,8 +49,82 @@ func New() DAO {
return &dao{}
}
var allowedMaps = map[string]interface{}{
strings.ToLower(rbac.ActionPull.String()): struct{}{},
strings.ToLower(rbac.ActionCreate.String()): struct{}{},
strings.ToLower(rbac.ActionDelete.String()): struct{}{},
}
type dao struct{}
// Purge delete expired audit log
func (*dao) Purge(ctx context.Context, retentionHour int, includeOperations []string, dryRun bool) (int64, error) {
ormer, err := orm.FromContext(ctx)
if err != nil {
return 0, err
}
if dryRun {
return dryRunPurge(ormer, retentionHour, includeOperations)
}
sql := "DELETE FROM audit_log WHERE op_time < NOW() - ? * interval '1 hour' "
filterOps := permitOps(includeOperations)
if len(filterOps) == 0 {
log.Infof("no operation selected, skip to purge audit log")
return 0, nil
}
sql = sql + "AND lower(operation) IN ('" + strings.Join(filterOps, "','") + "')"
log.Debugf("the sql is %v", sql)
r, err := ormer.Raw(sql, retentionHour).Exec()
if err != nil {
log.Errorf("failed to purge audit log, error %v", err)
return 0, err
}
delRows, rErr := r.RowsAffected()
if rErr != nil {
log.Errorf("failed to purge audit log, error %v", rErr)
return 0, rErr
}
log.Infof("purged %d audit logs in the database", delRows)
return delRows, err
}
func dryRunPurge(ormer beegorm.Ormer, retentionHour int, includeOperations []string) (int64, error) {
sql := "SELECT count(1) cnt FROM audit_log WHERE op_time < NOW() - ? * interval '1 hour' "
filterOps := permitOps(includeOperations)
if len(filterOps) == 0 {
log.Infof("[DRYRUN]no operation selected, skip to purge audit log")
return 0, nil
}
sql = sql + "AND lower(operation) IN ('" + strings.Join(filterOps, "','") + "')"
log.Debugf("the sql is %v", sql)
var cnt int64
err := ormer.Raw(sql, retentionHour).QueryRow(&cnt)
if err != nil {
log.Errorf("failed to dry run purge audit log, error %v", err)
return 0, err
}
log.Infof("[DRYRUN]purged %d audit logs in the database", cnt)
return cnt, nil
}
// permitOps filter not allowed operation, if no operation specified, purge pull operation
func permitOps(includeOperations []string) []string {
if includeOperations == nil {
return nil
}
var filterOps []string
for _, ops := range includeOperations {
ops := strings.ToLower(ops)
if _, exist := allowedMaps[ops]; exist {
filterOps = append(filterOps, ops)
}
}
return filterOps
}
// Count ...
func (d *dao) Count(ctx context.Context, query *q.Query) (int64, error) {
qs, err := orm.QuerySetterForCount(ctx, &model.AuditLog{}, query)

View File

@ -16,7 +16,7 @@ package dao
import (
"context"
"testing"
"reflect"
beegoorm "github.com/beego/beego/orm"
common_dao "github.com/goharbor/harbor/src/common/dao"
@ -25,6 +25,8 @@ import (
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/audit/model"
"github.com/stretchr/testify/suite"
"testing"
"time"
)
type daoTestSuite struct {
@ -43,14 +45,18 @@ func (d *daoTestSuite) SetupSuite() {
ResourceType: "artifact",
Resource: "library/test-audit",
Username: "admin",
OpTime: time.Now().AddDate(0, 0, -8),
})
d.Require().Nil(err)
d.auditID = artifactID
}
func (d *daoTestSuite) TearDownSuite() {
err := d.dao.Delete(d.ctx, d.auditID)
ormer, err := orm.FromContext(d.ctx)
d.Require().Nil(err)
_, err = ormer.Raw("delete from audit_log").Exec()
d.Require().Nil(err)
}
func (d *daoTestSuite) TestCount() {
@ -158,6 +164,82 @@ func (d *daoTestSuite) TestDelete() {
d.Equal(errors.NotFoundCode, e.Code)
}
func (d *daoTestSuite) TestPurge() {
result, err := d.dao.Purge(d.ctx, 24*30, []string{"Create"}, true)
d.Require().Nil(err)
d.Require().Equal(int64(0), result)
result1, err := d.dao.Purge(d.ctx, 24*7, []string{"Create"}, true)
d.Require().Nil(err)
d.Require().Equal(int64(1), result1)
}
func TestDaoTestSuite(t *testing.T) {
suite.Run(t, &daoTestSuite{})
}
func (d *daoTestSuite) Test_dao_Purge() {
d.ctx = orm.NewContext(nil, beegoorm.NewOrm())
_, err := d.dao.Create(d.ctx, &model.AuditLog{
Operation: "Delete",
ResourceType: "artifact",
Resource: "library/test-audit",
Username: "admin",
OpTime: time.Now().AddDate(0, 0, -8),
})
d.Require().Nil(err)
type args struct {
ctx context.Context
retentionHour int
includeOperations []string
dryRun bool
}
tests := []struct {
name string
args args
want int64
wantErr bool
}{
{"dry run 1 month", args{d.ctx, 24 * 30, []string{"create", "delete", "pull"}, true}, int64(0), false},
{"dry run 1 week", args{d.ctx, 24 * 7, []string{"create", "delete", "pull"}, true}, int64(2), false},
{"dry run delete run 1 week", args{d.ctx, 24 * 7, []string{"Delete"}, true}, int64(1), false},
{"delete run 1 week", args{d.ctx, 24 * 7, []string{"Delete"}, false}, int64(1), false},
}
for _, tt := range tests {
d.Run(tt.name, func() {
got, err := d.dao.Purge(tt.args.ctx, tt.args.retentionHour, tt.args.includeOperations, tt.args.dryRun)
if tt.wantErr {
d.Require().NotNil(err)
} else {
d.Require().Nil(err)
}
d.Require().Equal(tt.want, got)
})
}
}
func Test_filterOps(t *testing.T) {
type args struct {
includeOperations []string
}
tests := []struct {
name string
args args
want []string
}{
{"normal", args{[]string{"delete", "create", "pull"}}, []string{"delete", "create", "pull"}},
{"upper cased", args{[]string{"Delete", "Create", "Pull"}}, []string{"delete", "create", "pull"}},
{"mixed with not allowed", args{[]string{"Delete", "Create", "not_allowed_operation", "Pull"}}, []string{"delete", "create", "pull"}},
{"empty", args{[]string{}}, nil},
{"all not allowed", args{[]string{"destroy", "insert", "query"}}, nil},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := permitOps(tt.args.includeOperations); !reflect.DeepEqual(got, tt.want) {
t.Errorf("permitOps() = %v, want %v", got, tt.want)
}
})
}
}

View File

@ -36,6 +36,8 @@ type Manager interface {
Create(ctx context.Context, audit *model.AuditLog) (id int64, err error)
// Delete the audit log specified by ID
Delete(ctx context.Context, id int64) (err error)
// Purge delete the audit log with retention hours
Purge(ctx context.Context, retentionHour int, includeOperations []string, dryRun bool) (int64, error)
}
// New returns a default implementation of Manager
@ -69,6 +71,11 @@ func (m *manager) Create(ctx context.Context, audit *model.AuditLog) (int64, err
return m.dao.Create(ctx, audit)
}
// Purge ...
func (m *manager) Purge(ctx context.Context, retentionHour int, includeOperations []string, dryRun bool) (int64, error) {
return m.dao.Purge(ctx, retentionHour, includeOperations, dryRun)
}
// Delete ...
func (m *manager) Delete(ctx context.Context, id int64) error {
return m.dao.Delete(ctx, id)

View File

@ -15,54 +15,28 @@
package audit
import (
"context"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/audit/model"
mockDAO "github.com/goharbor/harbor/src/testing/pkg/audit/dao"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"testing"
)
type fakeDao struct {
mock.Mock
}
func (f *fakeDao) Count(ctx context.Context, query *q.Query) (int64, error) {
args := f.Called()
return int64(args.Int(0)), args.Error(1)
}
func (f *fakeDao) List(ctx context.Context, query *q.Query) ([]*model.AuditLog, error) {
args := f.Called()
return args.Get(0).([]*model.AuditLog), args.Error(1)
}
func (f *fakeDao) Get(ctx context.Context, id int64) (*model.AuditLog, error) {
args := f.Called()
return args.Get(0).(*model.AuditLog), args.Error(1)
}
func (f *fakeDao) Create(ctx context.Context, repository *model.AuditLog) (int64, error) {
args := f.Called()
return int64(args.Int(0)), args.Error(1)
}
func (f *fakeDao) Delete(ctx context.Context, id int64) error {
args := f.Called()
return args.Error(0)
}
type managerTestSuite struct {
suite.Suite
mgr *manager
dao *fakeDao
dao *mockDAO.DAO
}
func (m *managerTestSuite) SetupTest() {
m.dao = &fakeDao{}
m.dao = &mockDAO.DAO{}
m.mgr = &manager{
dao: m.dao,
}
}
func (m *managerTestSuite) TestCount() {
m.dao.On("Count", mock.Anything).Return(1, nil)
m.dao.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil)
total, err := m.mgr.Count(nil, nil)
m.Require().Nil(err)
m.Equal(int64(1), total)
@ -74,7 +48,7 @@ func (m *managerTestSuite) TestList() {
Resource: "library/hello-world",
ResourceType: "artifact",
}
m.dao.On("List", mock.Anything).Return([]*model.AuditLog{audit}, nil)
m.dao.On("List", mock.Anything, mock.Anything).Return([]*model.AuditLog{audit}, nil)
auditLogs, err := m.mgr.List(nil, nil)
m.Require().Nil(err)
m.Equal(1, len(auditLogs))
@ -87,7 +61,7 @@ func (m *managerTestSuite) TestGet() {
Resource: "library/hello-world",
ResourceType: "artifact",
}
m.dao.On("Get", mock.Anything).Return(audit, nil)
m.dao.On("Get", mock.Anything, mock.Anything).Return(audit, nil)
au, err := m.mgr.Get(nil, 1)
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())
@ -96,7 +70,7 @@ func (m *managerTestSuite) TestGet() {
}
func (m *managerTestSuite) TestCreate() {
m.dao.On("Create", mock.Anything).Return(1, nil)
m.dao.On("Create", mock.Anything, mock.Anything).Return(int64(1), nil)
id, err := m.mgr.Create(nil, &model.AuditLog{
ProjectID: 1,
Resource: "library/hello-world",
@ -108,7 +82,7 @@ func (m *managerTestSuite) TestCreate() {
}
func (m *managerTestSuite) TestDelete() {
m.dao.On("Delete", mock.Anything).Return(nil)
m.dao.On("Delete", mock.Anything, mock.Anything).Return(nil)
err := m.mgr.Delete(nil, 1)
m.Require().Nil(err)
m.dao.AssertExpectations(m.T())

View File

@ -28,3 +28,4 @@ package controller
//go:generate mockery --case snake --dir ../../controller/config --name Controller --output ./config --outpkg config
//go:generate mockery --case snake --dir ../../controller/user --name Controller --output ./user --outpkg user
//go:generate mockery --case snake --dir ../../controller/repository --name Controller --output ./repository --outpkg repository
//go:generate mockery --case snake --dir ../../controller/purge --name Controller --output ./purge --outpkg purge

View File

@ -0,0 +1,36 @@
// Code generated by mockery v2.1.0. DO NOT EDIT.
package purge
import (
context "context"
purge "github.com/goharbor/harbor/src/controller/purge"
mock "github.com/stretchr/testify/mock"
)
// Controller is an autogenerated mock type for the Controller type
type Controller struct {
mock.Mock
}
// Start provides a mock function with given fields: ctx, policy, trigger
func (_m *Controller) Start(ctx context.Context, policy purge.JobPolicy, trigger string) (int64, error) {
ret := _m.Called(ctx, policy, trigger)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, purge.JobPolicy, string) int64); ok {
r0 = rf(ctx, policy, trigger)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, purge.JobPolicy, string) error); ok {
r1 = rf(ctx, policy, trigger)
} else {
r1 = ret.Error(1)
}
return r0, r1
}

View File

@ -0,0 +1,141 @@
// Code generated by mockery v2.1.0. DO NOT EDIT.
package dao
import (
context "context"
mock "github.com/stretchr/testify/mock"
model "github.com/goharbor/harbor/src/pkg/audit/model"
q "github.com/goharbor/harbor/src/lib/q"
)
// DAO is an autogenerated mock type for the DAO type
type DAO struct {
mock.Mock
}
// Count provides a mock function with given fields: ctx, query
func (_m *DAO) Count(ctx context.Context, query *q.Query) (int64, error) {
ret := _m.Called(ctx, query)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok {
r0 = rf(ctx, query)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
r1 = rf(ctx, query)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Create provides a mock function with given fields: ctx, access
func (_m *DAO) Create(ctx context.Context, access *model.AuditLog) (int64, error) {
ret := _m.Called(ctx, access)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, *model.AuditLog) int64); ok {
r0 = rf(ctx, access)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *model.AuditLog) error); ok {
r1 = rf(ctx, access)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Delete provides a mock function with given fields: ctx, id
func (_m *DAO) Delete(ctx context.Context, id int64) error {
ret := _m.Called(ctx, id)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok {
r0 = rf(ctx, id)
} else {
r0 = ret.Error(0)
}
return r0
}
// Get provides a mock function with given fields: ctx, id
func (_m *DAO) Get(ctx context.Context, id int64) (*model.AuditLog, error) {
ret := _m.Called(ctx, id)
var r0 *model.AuditLog
if rf, ok := ret.Get(0).(func(context.Context, int64) *model.AuditLog); ok {
r0 = rf(ctx, id)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.AuditLog)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
r1 = rf(ctx, id)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// List provides a mock function with given fields: ctx, query
func (_m *DAO) List(ctx context.Context, query *q.Query) ([]*model.AuditLog, error) {
ret := _m.Called(ctx, query)
var r0 []*model.AuditLog
if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*model.AuditLog); ok {
r0 = rf(ctx, query)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.AuditLog)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
r1 = rf(ctx, query)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Purge provides a mock function with given fields: ctx, retentionHour, includeOperations, dryRun
func (_m *DAO) Purge(ctx context.Context, retentionHour int, includeOperations []string, dryRun bool) (int64, error) {
ret := _m.Called(ctx, retentionHour, includeOperations, dryRun)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, int, []string, bool) int64); ok {
r0 = rf(ctx, retentionHour, includeOperations, dryRun)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int, []string, bool) error); ok {
r1 = rf(ctx, retentionHour, includeOperations, dryRun)
} else {
r1 = ret.Error(1)
}
return r0, r1
}

View File

@ -0,0 +1,140 @@
// Code generated by mockery v2.1.0. DO NOT EDIT.
package audit
import (
context "context"
model "github.com/goharbor/harbor/src/pkg/audit/model"
mock "github.com/stretchr/testify/mock"
q "github.com/goharbor/harbor/src/lib/q"
)
// Manager is an autogenerated mock type for the Manager type
type Manager struct {
mock.Mock
}
// Count provides a mock function with given fields: ctx, query
func (_m *Manager) Count(ctx context.Context, query *q.Query) (int64, error) {
ret := _m.Called(ctx, query)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok {
r0 = rf(ctx, query)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
r1 = rf(ctx, query)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Create provides a mock function with given fields: ctx, _a1
func (_m *Manager) Create(ctx context.Context, _a1 *model.AuditLog) (int64, error) {
ret := _m.Called(ctx, _a1)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, *model.AuditLog) int64); ok {
r0 = rf(ctx, _a1)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *model.AuditLog) error); ok {
r1 = rf(ctx, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Delete provides a mock function with given fields: ctx, id
func (_m *Manager) Delete(ctx context.Context, id int64) error {
ret := _m.Called(ctx, id)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok {
r0 = rf(ctx, id)
} else {
r0 = ret.Error(0)
}
return r0
}
// Get provides a mock function with given fields: ctx, id
func (_m *Manager) Get(ctx context.Context, id int64) (*model.AuditLog, error) {
ret := _m.Called(ctx, id)
var r0 *model.AuditLog
if rf, ok := ret.Get(0).(func(context.Context, int64) *model.AuditLog); ok {
r0 = rf(ctx, id)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.AuditLog)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
r1 = rf(ctx, id)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// List provides a mock function with given fields: ctx, query
func (_m *Manager) List(ctx context.Context, query *q.Query) ([]*model.AuditLog, error) {
ret := _m.Called(ctx, query)
var r0 []*model.AuditLog
if rf, ok := ret.Get(0).(func(context.Context, *q.Query) []*model.AuditLog); ok {
r0 = rf(ctx, query)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.AuditLog)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
r1 = rf(ctx, query)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Purge provides a mock function with given fields: ctx, retentionHour, includeOperations, dryRun
func (_m *Manager) Purge(ctx context.Context, retentionHour int, includeOperations []string, dryRun bool) (int64, error) {
ret := _m.Called(ctx, retentionHour, includeOperations, dryRun)
var r0 int64
if rf, ok := ret.Get(0).(func(context.Context, int, []string, bool) int64); ok {
r0 = rf(ctx, retentionHour, includeOperations, dryRun)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int, []string, bool) error); ok {
r1 = rf(ctx, retentionHour, includeOperations, dryRun)
} else {
r1 = ret.Error(1)
}
return r0, r1
}

View File

@ -56,6 +56,8 @@ package pkg
//go:generate mockery --case snake --dir ../../pkg/accessory/model --name Accessory --output ./accessory/model --outpkg model
//go:generate mockery --case snake --dir ../../pkg/accessory/dao --name DAO --output ./accessory/dao --outpkg dao
//go:generate mockery --case snake --dir ../../pkg/accessory --name Manager --output ./accessory --outpkg accessory
//go:generate mockery --case snake --dir ../../pkg/audit/dao --name DAO --output ./audit/dao --outpkg dao
//go:generate mockery --case snake --dir ../../pkg/audit --name Manager --output ./audit --outpkg audit
//go:generate mockery --case snake --dir ../../pkg/systemartifact --name Manager --output ./systemartifact --outpkg systemartifact
//go:generate mockery --case snake --dir ../../pkg/systemartifact/ --name Selector --output ./systemartifact/cleanup --outpkg cleanup
//go:generate mockery --case snake --dir ../../pkg/systemartifact/dao --name DAO --output ./systemartifact/dao --outpkg dao