From df237a5b17cba27c3313d3dde9a309fd3caad7c4 Mon Sep 17 00:00:00 2001 From: wang yan Date: Wed, 4 Mar 2020 10:45:17 +0800 Subject: [PATCH] add audit logs API 1, add API entry for get audit logs 2. add audit log manager to hanlder CRUD Use the new format of audit log to cover differernt resource, artifact/tag/repostory/project Signed-off-by: wang yan --- api/v2.0/swagger.yaml | 83 ++++++++++++ .../postgresql/0030_2.0.0_schema.up.sql | 11 ++ src/pkg/audit/dao/dao.go | 127 ++++++++++++++++++ src/pkg/audit/dao/dao_test.go | 119 ++++++++++++++++ src/pkg/audit/manager.go | 75 +++++++++++ src/pkg/audit/manager_test.go | 119 ++++++++++++++++ src/pkg/audit/model/model.go | 26 ++++ src/server/v2.0/handler/auditlog.go | 76 +++++++++++ src/server/v2.0/handler/auditlog_test.go | 3 + src/server/v2.0/handler/handler.go | 1 + 10 files changed, 640 insertions(+) create mode 100644 src/pkg/audit/dao/dao.go create mode 100644 src/pkg/audit/dao/dao_test.go create mode 100644 src/pkg/audit/manager.go create mode 100644 src/pkg/audit/manager_test.go create mode 100644 src/pkg/audit/model/model.go create mode 100644 src/server/v2.0/handler/auditlog.go create mode 100644 src/server/v2.0/handler/auditlog_test.go diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 2557e7c3d..187740db3 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -499,6 +499,68 @@ paths: $ref: '#/responses/409' '500': $ref: '#/responses/500' + /audit-logs: + get: + summary: Get recent logs of the projects which the user is a member of + description: | + This endpoint let user see the recent operation logs of the projects which he is member of + tags: + - auditlog + operationId: listAuditLogs + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/page' + - $ref: '#/parameters/pageSize' + - name: username + in: query + type: string + required: false + description: Username of the operator. + - name: resource + in: query + type: string + required: false + description: The identity of resource + - name: resource_type + in: query + type: string + required: false + description: The type of resource, artifact/tag/repository + - name: operation + in: query + type: string + required: false + description: The operation, create/delete + - name: begin_timestamp + in: query + type: string + required: false + description: The begin timestamp + - name: end_timestamp + in: query + type: string + required: false + description: The end timestamp + responses: + '200': + description: Success + headers: + X-Total-Count: + description: The total count of auditlogs + type: integer + Link: + description: Link refers to the previous page and next page + type: string + schema: + type: array + items: + $ref: '#/definitions/AuditLog' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '500': + $ref: '#/responses/500' parameters: requestId: name: X-Request-Id @@ -919,3 +981,24 @@ definitions: example: 'Critical': 5 'High': 5 + AuditLog: + type: object + properties: + id: + type: integer + description: The ID of the audit log entry. + username: + type: string + description: Username of the user in this log entry. + resource: + type: string + description: Name of the repository in this log entry. + resource_type: + type: string + description: Tag of the repository in this log entry. + operation: + type: string + description: The operation against the repository in this log entry. + op_time: + type: string + description: The time when this operation is triggered. diff --git a/make/migrations/postgresql/0030_2.0.0_schema.up.sql b/make/migrations/postgresql/0030_2.0.0_schema.up.sql index cb034c713..8ce0d9439 100644 --- a/make/migrations/postgresql/0030_2.0.0_schema.up.sql +++ b/make/migrations/postgresql/0030_2.0.0_schema.up.sql @@ -173,3 +173,14 @@ CREATE TABLE artifact_2 push_time timestamp, CONSTRAINT unique_artifact_2 UNIQUE (project_id, repo, tag) ); + +CREATE TABLE audit_log +( + id SERIAL PRIMARY KEY NOT NULL, + project_id int NOT NULL, + operation varchar(20) NOT NULL, + resource_type varchar(255) NOT NULL, + resource varchar(1024) NOT NULL, + username varchar(255) NOT NULL, + op_time timestamp default CURRENT_TIMESTAMP +); diff --git a/src/pkg/audit/dao/dao.go b/src/pkg/audit/dao/dao.go new file mode 100644 index 000000000..90141d4e0 --- /dev/null +++ b/src/pkg/audit/dao/dao.go @@ -0,0 +1,127 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "context" + ierror "github.com/goharbor/harbor/src/internal/error" + "github.com/goharbor/harbor/src/internal/orm" + "github.com/goharbor/harbor/src/pkg/audit/model" + "github.com/goharbor/harbor/src/pkg/q" +) + +// DAO is the data access object for audit log +type DAO interface { + // Create the audit log + Create(ctx context.Context, access *model.AuditLog) (id int64, err error) + // Count returns the total count of audit logs according to the query + Count(ctx context.Context, query *q.Query) (total int64, err error) + // List audit logs according to the query + List(ctx context.Context, query *q.Query) (access []*model.AuditLog, err error) + // Get the audit log specified by ID + Get(ctx context.Context, id int64) (access *model.AuditLog, err error) + // Delete the audit log specified by ID + Delete(ctx context.Context, id int64) (err error) +} + +// New returns an instance of the default DAO +func New() DAO { + return &dao{} +} + +type dao struct{} + +// Count ... +func (d *dao) Count(ctx context.Context, query *q.Query) (int64, error) { + if query != nil { + // ignore the page number and size + query = &q.Query{ + Keywords: query.Keywords, + } + } + qs, err := orm.QuerySetter(ctx, &model.AuditLog{}, query) + if err != nil { + return 0, err + } + return qs.Count() +} + +// List ... +func (d *dao) List(ctx context.Context, query *q.Query) ([]*model.AuditLog, error) { + audit := []*model.AuditLog{} + qs, err := orm.QuerySetter(ctx, &model.AuditLog{}, query) + qs = qs.OrderBy("-op_time") + if err != nil { + return nil, err + } + if _, err = qs.All(&audit); err != nil { + return nil, err + } + return audit, nil +} + +// Get ... +func (d *dao) Get(ctx context.Context, id int64) (*model.AuditLog, error) { + audit := &model.AuditLog{ + ID: id, + } + ormer, err := orm.FromContext(ctx) + if err != nil { + return nil, err + } + if err := ormer.Read(audit); err != nil { + if e := orm.AsNotFoundError(err, "audit %d not found", id); e != nil { + err = e + } + return nil, err + } + return audit, nil +} + +// Create ... +func (d *dao) Create(ctx context.Context, audit *model.AuditLog) (int64, error) { + ormer, err := orm.FromContext(ctx) + if err != nil { + return 0, err + } + // the max length of username in database is 255, replace the last + // three charaters with "..." if the length is greater than 256 + if len(audit.Username) > 255 { + audit.Username = audit.Username[:252] + "..." + } + id, err := ormer.Insert(audit) + if err != nil { + return 0, err + } + return id, err +} + +// Delete ... +func (d *dao) Delete(ctx context.Context, id int64) error { + ormer, err := orm.FromContext(ctx) + if err != nil { + return err + } + n, err := ormer.Delete(&model.AuditLog{ + ID: id, + }) + if err != nil { + return err + } + if n == 0 { + return ierror.NotFoundError(nil).WithMessage("access %d not found", id) + } + return nil +} diff --git a/src/pkg/audit/dao/dao_test.go b/src/pkg/audit/dao/dao_test.go new file mode 100644 index 000000000..f28e5a3ae --- /dev/null +++ b/src/pkg/audit/dao/dao_test.go @@ -0,0 +1,119 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "context" + "errors" + beegoorm "github.com/astaxie/beego/orm" + common_dao "github.com/goharbor/harbor/src/common/dao" + ierror "github.com/goharbor/harbor/src/internal/error" + "github.com/goharbor/harbor/src/internal/orm" + "github.com/goharbor/harbor/src/pkg/audit/model" + "github.com/goharbor/harbor/src/pkg/q" + "github.com/stretchr/testify/suite" + "testing" +) + +type daoTestSuite struct { + suite.Suite + dao DAO + auditID int64 + ctx context.Context +} + +func (d *daoTestSuite) SetupSuite() { + d.dao = New() + common_dao.PrepareTestForPostgresSQL() + d.ctx = orm.NewContext(nil, beegoorm.NewOrm()) + artifactID, err := d.dao.Create(d.ctx, &model.AuditLog{ + Operation: "Create", + ResourceType: "artifact", + Resource: "library/hello-world", + Username: "admin", + }) + d.Require().Nil(err) + d.auditID = artifactID +} + +func (d *daoTestSuite) TearDownSuite() { + err := d.dao.Delete(d.ctx, d.auditID) + d.Require().Nil(err) +} + +func (d *daoTestSuite) TestCount() { + total, err := d.dao.Count(d.ctx, nil) + d.Require().Nil(err) + d.True(total > 0) + total, err = d.dao.Count(d.ctx, &q.Query{ + Keywords: map[string]interface{}{ + "ResourceType": "artifact", + }, + }) + d.Require().Nil(err) + d.Equal(int64(1), total) +} + +func (d *daoTestSuite) TestList() { + // nil query + audits, err := d.dao.List(d.ctx, nil) + d.Require().Nil(err) + + // query by repository ID and name + audits, err = d.dao.List(d.ctx, &q.Query{ + Keywords: map[string]interface{}{ + "ResourceType": "artifact", + }, + }) + d.Require().Nil(err) + d.Require().Equal(1, len(audits)) + d.Equal("admin", audits[0].Username) +} + +func (d *daoTestSuite) TestGet() { + // get the non-exist tag + _, err := d.dao.Get(d.ctx, 10000) + d.Require().NotNil(err) + d.True(ierror.IsErr(err, ierror.NotFoundCode)) + + audit, err := d.dao.Get(d.ctx, d.auditID) + d.Require().Nil(err) + d.Require().NotNil(audit) + d.Equal(d.auditID, audit.ID) +} + +func (d *daoTestSuite) TestCreate() { + // conflict + audit := &model.AuditLog{ + Operation: "Create", + ResourceType: "tag", + Resource: "library/hello-world", + Username: "admin", + } + _, err := d.dao.Create(d.ctx, audit) + d.Require().Nil(err) +} + +func (d *daoTestSuite) TestDelete() { + err := d.dao.Delete(d.ctx, 10000) + d.Require().NotNil(err) + var e *ierror.Error + d.Require().True(errors.As(err, &e)) + d.Equal(ierror.NotFoundCode, e.Code) +} + +func TestDaoTestSuite(t *testing.T) { + suite.Run(t, &daoTestSuite{}) +} diff --git a/src/pkg/audit/manager.go b/src/pkg/audit/manager.go new file mode 100644 index 000000000..4cde9a250 --- /dev/null +++ b/src/pkg/audit/manager.go @@ -0,0 +1,75 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package audit + +import ( + "context" + "github.com/goharbor/harbor/src/pkg/audit/dao" + "github.com/goharbor/harbor/src/pkg/audit/model" + "github.com/goharbor/harbor/src/pkg/q" +) + +// Mgr is the global audit log manager instance +var Mgr = New() + +// Manager is used for audit log management +type Manager interface { + // Count returns the total count of audit logs according to the query + Count(ctx context.Context, query *q.Query) (total int64, err error) + // List audit logs according to the query + List(ctx context.Context, query *q.Query) (audits []*model.AuditLog, err error) + // Get the audit log specified by ID + Get(ctx context.Context, id int64) (audit *model.AuditLog, err error) + // Create the audit log + Create(ctx context.Context, audit *model.AuditLog) (id int64, err error) + // Delete the audit log specified by ID + Delete(ctx context.Context, id int64) (err error) +} + +// New returns a default implementation of Manager +func New() Manager { + return &manager{ + dao: dao.New(), + } +} + +type manager struct { + dao dao.DAO +} + +// Count ... +func (m *manager) Count(ctx context.Context, query *q.Query) (int64, error) { + return m.dao.Count(ctx, query) +} + +// List ... +func (m *manager) List(ctx context.Context, query *q.Query) ([]*model.AuditLog, error) { + return m.dao.List(ctx, query) +} + +// Get ... +func (m *manager) Get(ctx context.Context, id int64) (*model.AuditLog, error) { + return m.dao.Get(ctx, id) +} + +// Create ... +func (m *manager) Create(ctx context.Context, audit *model.AuditLog) (int64, error) { + return m.dao.Create(ctx, audit) +} + +// Delete ... +func (m *manager) Delete(ctx context.Context, id int64) error { + return m.dao.Delete(ctx, id) +} diff --git a/src/pkg/audit/manager_test.go b/src/pkg/audit/manager_test.go new file mode 100644 index 000000000..394c74934 --- /dev/null +++ b/src/pkg/audit/manager_test.go @@ -0,0 +1,119 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package audit + +import ( + "context" + "github.com/goharbor/harbor/src/pkg/audit/model" + "github.com/goharbor/harbor/src/pkg/q" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "testing" +) + +type fakeDao struct { + mock.Mock +} + +func (f *fakeDao) Count(ctx context.Context, query *q.Query) (int64, error) { + args := f.Called() + return int64(args.Int(0)), args.Error(1) +} +func (f *fakeDao) List(ctx context.Context, query *q.Query) ([]*model.AuditLog, error) { + args := f.Called() + return args.Get(0).([]*model.AuditLog), args.Error(1) +} +func (f *fakeDao) Get(ctx context.Context, id int64) (*model.AuditLog, error) { + args := f.Called() + return args.Get(0).(*model.AuditLog), args.Error(1) +} +func (f *fakeDao) Create(ctx context.Context, repository *model.AuditLog) (int64, error) { + args := f.Called() + return int64(args.Int(0)), args.Error(1) +} +func (f *fakeDao) Delete(ctx context.Context, id int64) error { + args := f.Called() + return args.Error(0) +} + +type managerTestSuite struct { + suite.Suite + mgr *manager + dao *fakeDao +} + +func (m *managerTestSuite) SetupTest() { + m.dao = &fakeDao{} + m.mgr = &manager{ + dao: m.dao, + } +} + +func (m *managerTestSuite) TestCount() { + m.dao.On("Count", mock.Anything).Return(1, nil) + total, err := m.mgr.Count(nil, nil) + m.Require().Nil(err) + m.Equal(int64(1), total) +} + +func (m *managerTestSuite) TestList() { + audit := &model.AuditLog{ + ProjectID: 1, + Resource: "library/hello-world", + ResourceType: "artifact", + } + m.dao.On("List", mock.Anything).Return([]*model.AuditLog{audit}, nil) + auditLogs, err := m.mgr.List(nil, nil) + m.Require().Nil(err) + m.Equal(1, len(auditLogs)) + m.Equal(audit.Resource, auditLogs[0].Resource) +} + +func (m *managerTestSuite) TestGet() { + audit := &model.AuditLog{ + ProjectID: 1, + Resource: "library/hello-world", + ResourceType: "artifact", + } + m.dao.On("Get", mock.Anything).Return(audit, nil) + au, err := m.mgr.Get(nil, 1) + m.Require().Nil(err) + m.dao.AssertExpectations(m.T()) + m.Require().NotNil(au) + m.Equal(audit.Resource, au.Resource) +} + +func (m *managerTestSuite) TestCreate() { + m.dao.On("Create", mock.Anything).Return(1, nil) + id, err := m.mgr.Create(nil, &model.AuditLog{ + ProjectID: 1, + Resource: "library/hello-world", + ResourceType: "artifact", + }) + m.Require().Nil(err) + m.dao.AssertExpectations(m.T()) + m.Equal(int64(1), id) +} + +func (m *managerTestSuite) TestDelete() { + m.dao.On("Delete", mock.Anything).Return(nil) + err := m.mgr.Delete(nil, 1) + m.Require().Nil(err) + m.dao.AssertExpectations(m.T()) +} + +func TestManager(t *testing.T) { + suite.Run(t, &managerTestSuite{}) +} diff --git a/src/pkg/audit/model/model.go b/src/pkg/audit/model/model.go new file mode 100644 index 000000000..22815fbc4 --- /dev/null +++ b/src/pkg/audit/model/model.go @@ -0,0 +1,26 @@ +package model + +import ( + beego_orm "github.com/astaxie/beego/orm" + "time" +) + +func init() { + beego_orm.RegisterModel(&AuditLog{}) +} + +// AuditLog ... +type AuditLog struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + ProjectID int `orm:"column(project_id)" json:"project_id"` + Operation string `orm:"column(operation)" json:"operation"` + ResourceType string `orm:"column(resource_type)" json:"resource_type"` + Resource string `orm:"column(resource)" json:"resource"` + Username string `orm:"column(username)" json:"username"` + OpTime time.Time `orm:"column(op_time)" json:"op_time"` +} + +// TableName for audit log +func (a *AuditLog) TableName() string { + return "audit_log" +} diff --git a/src/server/v2.0/handler/auditlog.go b/src/server/v2.0/handler/auditlog.go new file mode 100644 index 000000000..b69e4c4f9 --- /dev/null +++ b/src/server/v2.0/handler/auditlog.go @@ -0,0 +1,76 @@ +package handler + +import ( + "context" + "github.com/go-openapi/runtime/middleware" + "github.com/goharbor/harbor/src/pkg/audit" + "github.com/goharbor/harbor/src/pkg/q" + "github.com/goharbor/harbor/src/server/v2.0/models" + "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/auditlog" + operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/auditlog" +) + +func newAuditLogAPI() *auditlogAPI { + return &auditlogAPI{ + auditMgr: audit.Mgr, + } +} + +type auditlogAPI struct { + BaseAPI + auditMgr audit.Manager +} + +func (a *auditlogAPI) ListAuditLogs(ctx context.Context, params auditlog.ListAuditLogsParams) middleware.Responder { + // ToDo enable permission check + // if !a.HasPermission(ctx, rbac.ActionList, rbac.ResourceLog) { + // return a.SendError(ctx, ierror.ForbiddenError(nil)) + // } + keywords := make(map[string]interface{}) + query := &q.Query{ + Keywords: keywords, + } + // TODO support fuzzy match and start end time + if params.Username != nil { + query.Keywords["Username"] = *(params.Username) + } + if params.Operation != nil { + query.Keywords["Operation"] = *(params.Operation) + } + if params.Resource != nil { + query.Keywords["Resource"] = *(params.Resource) + } + if params.ResourceType != nil { + query.Keywords["ResourceType"] = *(params.ResourceType) + } + if params.Page != nil { + query.PageNumber = *(params.Page) + } + if params.PageSize != nil { + query.PageSize = *(params.PageSize) + } + total, err := a.auditMgr.Count(ctx, query) + if err != nil { + return a.SendError(ctx, err) + } + logs, err := a.auditMgr.List(ctx, query) + if err != nil { + return a.SendError(ctx, err) + } + + var auditLogs []*models.AuditLog + for _, log := range logs { + auditLogs = append(auditLogs, &models.AuditLog{ + ID: log.ID, + Resource: log.Resource, + ResourceType: log.ResourceType, + Username: log.Username, + Operation: log.Operation, + OpTime: log.OpTime.String(), + }) + } + return operation.NewListAuditLogsOK(). + WithXTotalCount(total). + WithLink(a.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()). + WithPayload(auditLogs) +} diff --git a/src/server/v2.0/handler/auditlog_test.go b/src/server/v2.0/handler/auditlog_test.go new file mode 100644 index 000000000..f73ac1780 --- /dev/null +++ b/src/server/v2.0/handler/auditlog_test.go @@ -0,0 +1,3 @@ +package handler + +// ToDo add api tests diff --git a/src/server/v2.0/handler/handler.go b/src/server/v2.0/handler/handler.go index dde53d47f..86dfc1345 100644 --- a/src/server/v2.0/handler/handler.go +++ b/src/server/v2.0/handler/handler.go @@ -26,6 +26,7 @@ func New() http.Handler { h, api, err := restapi.HandlerAPI(restapi.Config{ ArtifactAPI: newArtifactAPI(), RepositoryAPI: newRepositoryAPI(), + AuditlogAPI: newAuditLogAPI(), }) if err != nil { log.Fatal(err)