Merge pull request #10939 from wy65701436/access-log-mgr

add audit logs API

api/v2.0/swagger.yaml

@@ -499,6 +499,68 @@ paths:
           $ref: '#/responses/409'
         '500':
           $ref: '#/responses/500'
+  /audit-logs:
+    get:
+      summary: Get recent logs of the projects which the user is a member of
+      description: |
+        This endpoint let user see the recent operation logs of the projects which he is member of
+      tags:
+        - auditlog
+      operationId: listAuditLogs
+      parameters:
+        - $ref: '#/parameters/requestId'
+        - $ref: '#/parameters/page'
+        - $ref: '#/parameters/pageSize'
+        - name: username
+          in: query
+          type: string
+          required: false
+          description: Username of the operator.
+        - name: resource
+          in: query
+          type: string
+          required: false
+          description: The identity of resource
+        - name: resource_type
+          in: query
+          type: string
+          required: false
+          description: The type of resource, artifact/tag/repository
+        - name: operation
+          in: query
+          type: string
+          required: false
+          description: The operation, create/delete
+        - name: begin_timestamp
+          in: query
+          type: string
+          required: false
+          description: The begin timestamp
+        - name: end_timestamp
+          in: query
+          type: string
+          required: false
+          description: The end timestamp
+      responses:
+        '200':
+          description: Success
+          headers:
+            X-Total-Count:
+              description: The total count of auditlogs
+              type: integer
+            Link:
+              description: Link refers to the previous page and next page
+              type: string
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/AuditLog'
+        '400':
+          $ref: '#/responses/400'
+        '401':
+          $ref: '#/responses/401'
+        '500':
+          $ref: '#/responses/500'
 parameters:
   requestId:
     name: X-Request-Id
@@ -919,3 +981,24 @@ definitions:
         example:
           'Critical': 5
           'High': 5
+  AuditLog:
+    type: object
+    properties:
+      id:
+        type: integer
+        description: The ID of the audit log entry.
+      username:
+        type: string
+        description: Username of the user in this log entry.
+      resource:
+        type: string
+        description: Name of the repository in this log entry.
+      resource_type:
+        type: string
+        description: Tag of the repository in this log entry.
+      operation:
+        type: string
+        description: The operation against the repository in this log entry.
+      op_time:
+        type: string
+        description: The time when this operation is triggered.

make/migrations/postgresql/0030_2.0.0_schema.up.sql

@@ -173,3 +173,14 @@ CREATE TABLE artifact_2
   push_time     timestamp,
   CONSTRAINT unique_artifact_2 UNIQUE (project_id, repo, tag)
 );
+
+CREATE TABLE audit_log
+(
+ id             SERIAL PRIMARY KEY NOT NULL,
+ project_id     int NOT NULL,
+ operation      varchar(20) NOT NULL,
+ resource_type  varchar(255) NOT NULL,
+ resource       varchar(1024) NOT NULL,
+ username       varchar(255) NOT NULL,
+ op_time        timestamp default CURRENT_TIMESTAMP
+);

src/pkg/audit/dao/dao.go

@@ -0,0 +1,127 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"context"
+	ierror "github.com/goharbor/harbor/src/internal/error"
+	"github.com/goharbor/harbor/src/internal/orm"
+	"github.com/goharbor/harbor/src/pkg/audit/model"
+	"github.com/goharbor/harbor/src/pkg/q"
+)
+
+// DAO is the data access object for audit log
+type DAO interface {
+	// Create the audit log
+	Create(ctx context.Context, access *model.AuditLog) (id int64, err error)
+	// Count returns the total count of audit logs according to the query
+	Count(ctx context.Context, query *q.Query) (total int64, err error)
+	// List audit logs according to the query
+	List(ctx context.Context, query *q.Query) (access []*model.AuditLog, err error)
+	// Get the audit log specified by ID
+	Get(ctx context.Context, id int64) (access *model.AuditLog, err error)
+	// Delete the audit log specified by ID
+	Delete(ctx context.Context, id int64) (err error)
+}
+
+// New returns an instance of the default DAO
+func New() DAO {
+	return &dao{}
+}
+
+type dao struct{}
+
+// Count ...
+func (d *dao) Count(ctx context.Context, query *q.Query) (int64, error) {
+	if query != nil {
+		// ignore the page number and size
+		query = &q.Query{
+			Keywords: query.Keywords,
+		}
+	}
+	qs, err := orm.QuerySetter(ctx, &model.AuditLog{}, query)
+	if err != nil {
+		return 0, err
+	}
+	return qs.Count()
+}
+
+// List ...
+func (d *dao) List(ctx context.Context, query *q.Query) ([]*model.AuditLog, error) {
+	audit := []*model.AuditLog{}
+	qs, err := orm.QuerySetter(ctx, &model.AuditLog{}, query)
+	qs = qs.OrderBy("-op_time")
+	if err != nil {
+		return nil, err
+	}
+	if _, err = qs.All(&audit); err != nil {
+		return nil, err
+	}
+	return audit, nil
+}
+
+// Get ...
+func (d *dao) Get(ctx context.Context, id int64) (*model.AuditLog, error) {
+	audit := &model.AuditLog{
+		ID: id,
+	}
+	ormer, err := orm.FromContext(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if err := ormer.Read(audit); err != nil {
+		if e := orm.AsNotFoundError(err, "audit %d not found", id); e != nil {
+			err = e
+		}
+		return nil, err
+	}
+	return audit, nil
+}
+
+// Create ...
+func (d *dao) Create(ctx context.Context, audit *model.AuditLog) (int64, error) {
+	ormer, err := orm.FromContext(ctx)
+	if err != nil {
+		return 0, err
+	}
+	// the max length of username in database is 255, replace the last
+	// three charaters with "..." if the length is greater than 256
+	if len(audit.Username) > 255 {
+		audit.Username = audit.Username[:252] + "..."
+	}
+	id, err := ormer.Insert(audit)
+	if err != nil {
+		return 0, err
+	}
+	return id, err
+}
+
+// Delete ...
+func (d *dao) Delete(ctx context.Context, id int64) error {
+	ormer, err := orm.FromContext(ctx)
+	if err != nil {
+		return err
+	}
+	n, err := ormer.Delete(&model.AuditLog{
+		ID: id,
+	})
+	if err != nil {
+		return err
+	}
+	if n == 0 {
+		return ierror.NotFoundError(nil).WithMessage("access %d not found", id)
+	}
+	return nil
+}

src/pkg/audit/dao/dao_test.go

@@ -0,0 +1,119 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"context"
+	"errors"
+	beegoorm "github.com/astaxie/beego/orm"
+	common_dao "github.com/goharbor/harbor/src/common/dao"
+	ierror "github.com/goharbor/harbor/src/internal/error"
+	"github.com/goharbor/harbor/src/internal/orm"
+	"github.com/goharbor/harbor/src/pkg/audit/model"
+	"github.com/goharbor/harbor/src/pkg/q"
+	"github.com/stretchr/testify/suite"
+	"testing"
+)
+
+type daoTestSuite struct {
+	suite.Suite
+	dao     DAO
+	auditID int64
+	ctx     context.Context
+}
+
+func (d *daoTestSuite) SetupSuite() {
+	d.dao = New()
+	common_dao.PrepareTestForPostgresSQL()
+	d.ctx = orm.NewContext(nil, beegoorm.NewOrm())
+	artifactID, err := d.dao.Create(d.ctx, &model.AuditLog{
+		Operation:    "Create",
+		ResourceType: "artifact",
+		Resource:     "library/hello-world",
+		Username:     "admin",
+	})
+	d.Require().Nil(err)
+	d.auditID = artifactID
+}
+
+func (d *daoTestSuite) TearDownSuite() {
+	err := d.dao.Delete(d.ctx, d.auditID)
+	d.Require().Nil(err)
+}
+
+func (d *daoTestSuite) TestCount() {
+	total, err := d.dao.Count(d.ctx, nil)
+	d.Require().Nil(err)
+	d.True(total > 0)
+	total, err = d.dao.Count(d.ctx, &q.Query{
+		Keywords: map[string]interface{}{
+			"ResourceType": "artifact",
+		},
+	})
+	d.Require().Nil(err)
+	d.Equal(int64(1), total)
+}
+
+func (d *daoTestSuite) TestList() {
+	// nil query
+	audits, err := d.dao.List(d.ctx, nil)
+	d.Require().Nil(err)
+
+	// query by repository ID and name
+	audits, err = d.dao.List(d.ctx, &q.Query{
+		Keywords: map[string]interface{}{
+			"ResourceType": "artifact",
+		},
+	})
+	d.Require().Nil(err)
+	d.Require().Equal(1, len(audits))
+	d.Equal("admin", audits[0].Username)
+}
+
+func (d *daoTestSuite) TestGet() {
+	// get the non-exist tag
+	_, err := d.dao.Get(d.ctx, 10000)
+	d.Require().NotNil(err)
+	d.True(ierror.IsErr(err, ierror.NotFoundCode))
+
+	audit, err := d.dao.Get(d.ctx, d.auditID)
+	d.Require().Nil(err)
+	d.Require().NotNil(audit)
+	d.Equal(d.auditID, audit.ID)
+}
+
+func (d *daoTestSuite) TestCreate() {
+	// conflict
+	audit := &model.AuditLog{
+		Operation:    "Create",
+		ResourceType: "tag",
+		Resource:     "library/hello-world",
+		Username:     "admin",
+	}
+	_, err := d.dao.Create(d.ctx, audit)
+	d.Require().Nil(err)
+}
+
+func (d *daoTestSuite) TestDelete() {
+	err := d.dao.Delete(d.ctx, 10000)
+	d.Require().NotNil(err)
+	var e *ierror.Error
+	d.Require().True(errors.As(err, &e))
+	d.Equal(ierror.NotFoundCode, e.Code)
+}
+
+func TestDaoTestSuite(t *testing.T) {
+	suite.Run(t, &daoTestSuite{})
+}

src/pkg/audit/manager.go

@@ -0,0 +1,75 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package audit
+
+import (
+	"context"
+	"github.com/goharbor/harbor/src/pkg/audit/dao"
+	"github.com/goharbor/harbor/src/pkg/audit/model"
+	"github.com/goharbor/harbor/src/pkg/q"
+)
+
+// Mgr is the global audit log manager instance
+var Mgr = New()
+
+// Manager is used for audit log management
+type Manager interface {
+	// Count returns the total count of audit logs according to the query
+	Count(ctx context.Context, query *q.Query) (total int64, err error)
+	// List audit logs according to the query
+	List(ctx context.Context, query *q.Query) (audits []*model.AuditLog, err error)
+	// Get the audit log specified by ID
+	Get(ctx context.Context, id int64) (audit *model.AuditLog, err error)
+	// Create the audit log
+	Create(ctx context.Context, audit *model.AuditLog) (id int64, err error)
+	// Delete the audit log specified by ID
+	Delete(ctx context.Context, id int64) (err error)
+}
+
+// New returns a default implementation of Manager
+func New() Manager {
+	return &manager{
+		dao: dao.New(),
+	}
+}
+
+type manager struct {
+	dao dao.DAO
+}
+
+// Count ...
+func (m *manager) Count(ctx context.Context, query *q.Query) (int64, error) {
+	return m.dao.Count(ctx, query)
+}
+
+// List ...
+func (m *manager) List(ctx context.Context, query *q.Query) ([]*model.AuditLog, error) {
+	return m.dao.List(ctx, query)
+}
+
+// Get ...
+func (m *manager) Get(ctx context.Context, id int64) (*model.AuditLog, error) {
+	return m.dao.Get(ctx, id)
+}
+
+// Create ...
+func (m *manager) Create(ctx context.Context, audit *model.AuditLog) (int64, error) {
+	return m.dao.Create(ctx, audit)
+}
+
+// Delete ...
+func (m *manager) Delete(ctx context.Context, id int64) error {
+	return m.dao.Delete(ctx, id)
+}

src/pkg/audit/manager_test.go

@@ -0,0 +1,119 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package audit
+
+import (
+	"context"
+	"github.com/goharbor/harbor/src/pkg/audit/model"
+	"github.com/goharbor/harbor/src/pkg/q"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/suite"
+	"testing"
+)
+
+type fakeDao struct {
+	mock.Mock
+}
+
+func (f *fakeDao) Count(ctx context.Context, query *q.Query) (int64, error) {
+	args := f.Called()
+	return int64(args.Int(0)), args.Error(1)
+}
+func (f *fakeDao) List(ctx context.Context, query *q.Query) ([]*model.AuditLog, error) {
+	args := f.Called()
+	return args.Get(0).([]*model.AuditLog), args.Error(1)
+}
+func (f *fakeDao) Get(ctx context.Context, id int64) (*model.AuditLog, error) {
+	args := f.Called()
+	return args.Get(0).(*model.AuditLog), args.Error(1)
+}
+func (f *fakeDao) Create(ctx context.Context, repository *model.AuditLog) (int64, error) {
+	args := f.Called()
+	return int64(args.Int(0)), args.Error(1)
+}
+func (f *fakeDao) Delete(ctx context.Context, id int64) error {
+	args := f.Called()
+	return args.Error(0)
+}
+
+type managerTestSuite struct {
+	suite.Suite
+	mgr *manager
+	dao *fakeDao
+}
+
+func (m *managerTestSuite) SetupTest() {
+	m.dao = &fakeDao{}
+	m.mgr = &manager{
+		dao: m.dao,
+	}
+}
+
+func (m *managerTestSuite) TestCount() {
+	m.dao.On("Count", mock.Anything).Return(1, nil)
+	total, err := m.mgr.Count(nil, nil)
+	m.Require().Nil(err)
+	m.Equal(int64(1), total)
+}
+
+func (m *managerTestSuite) TestList() {
+	audit := &model.AuditLog{
+		ProjectID:    1,
+		Resource:     "library/hello-world",
+		ResourceType: "artifact",
+	}
+	m.dao.On("List", mock.Anything).Return([]*model.AuditLog{audit}, nil)
+	auditLogs, err := m.mgr.List(nil, nil)
+	m.Require().Nil(err)
+	m.Equal(1, len(auditLogs))
+	m.Equal(audit.Resource, auditLogs[0].Resource)
+}
+
+func (m *managerTestSuite) TestGet() {
+	audit := &model.AuditLog{
+		ProjectID:    1,
+		Resource:     "library/hello-world",
+		ResourceType: "artifact",
+	}
+	m.dao.On("Get", mock.Anything).Return(audit, nil)
+	au, err := m.mgr.Get(nil, 1)
+	m.Require().Nil(err)
+	m.dao.AssertExpectations(m.T())
+	m.Require().NotNil(au)
+	m.Equal(audit.Resource, au.Resource)
+}
+
+func (m *managerTestSuite) TestCreate() {
+	m.dao.On("Create", mock.Anything).Return(1, nil)
+	id, err := m.mgr.Create(nil, &model.AuditLog{
+		ProjectID:    1,
+		Resource:     "library/hello-world",
+		ResourceType: "artifact",
+	})
+	m.Require().Nil(err)
+	m.dao.AssertExpectations(m.T())
+	m.Equal(int64(1), id)
+}
+
+func (m *managerTestSuite) TestDelete() {
+	m.dao.On("Delete", mock.Anything).Return(nil)
+	err := m.mgr.Delete(nil, 1)
+	m.Require().Nil(err)
+	m.dao.AssertExpectations(m.T())
+}
+
+func TestManager(t *testing.T) {
+	suite.Run(t, &managerTestSuite{})
+}

src/pkg/audit/model/model.go

@@ -0,0 +1,26 @@
+package model
+
+import (
+	beego_orm "github.com/astaxie/beego/orm"
+	"time"
+)
+
+func init() {
+	beego_orm.RegisterModel(&AuditLog{})
+}
+
+// AuditLog ...
+type AuditLog struct {
+	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
+	ProjectID    int       `orm:"column(project_id)" json:"project_id"`
+	Operation    string    `orm:"column(operation)" json:"operation"`
+	ResourceType string    `orm:"column(resource_type)"  json:"resource_type"`
+	Resource     string    `orm:"column(resource)" json:"resource"`
+	Username     string    `orm:"column(username)"  json:"username"`
+	OpTime       time.Time `orm:"column(op_time)" json:"op_time"`
+}
+
+// TableName for audit log
+func (a *AuditLog) TableName() string {
+	return "audit_log"
+}

src/server/v2.0/handler/auditlog.go

@@ -0,0 +1,76 @@
+package handler
+
+import (
+	"context"
+	"github.com/go-openapi/runtime/middleware"
+	"github.com/goharbor/harbor/src/pkg/audit"
+	"github.com/goharbor/harbor/src/pkg/q"
+	"github.com/goharbor/harbor/src/server/v2.0/models"
+	"github.com/goharbor/harbor/src/server/v2.0/restapi/operations/auditlog"
+	operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/auditlog"
+)
+
+func newAuditLogAPI() *auditlogAPI {
+	return &auditlogAPI{
+		auditMgr: audit.Mgr,
+	}
+}
+
+type auditlogAPI struct {
+	BaseAPI
+	auditMgr audit.Manager
+}
+
+func (a *auditlogAPI) ListAuditLogs(ctx context.Context, params auditlog.ListAuditLogsParams) middleware.Responder {
+	// ToDo enable permission check
+	// if !a.HasPermission(ctx, rbac.ActionList, rbac.ResourceLog) {
+	//	return a.SendError(ctx, ierror.ForbiddenError(nil))
+	// }
+	keywords := make(map[string]interface{})
+	query := &q.Query{
+		Keywords: keywords,
+	}
+	// TODO support fuzzy match and start end time
+	if params.Username != nil {
+		query.Keywords["Username"] = *(params.Username)
+	}
+	if params.Operation != nil {
+		query.Keywords["Operation"] = *(params.Operation)
+	}
+	if params.Resource != nil {
+		query.Keywords["Resource"] = *(params.Resource)
+	}
+	if params.ResourceType != nil {
+		query.Keywords["ResourceType"] = *(params.ResourceType)
+	}
+	if params.Page != nil {
+		query.PageNumber = *(params.Page)
+	}
+	if params.PageSize != nil {
+		query.PageSize = *(params.PageSize)
+	}
+	total, err := a.auditMgr.Count(ctx, query)
+	if err != nil {
+		return a.SendError(ctx, err)
+	}
+	logs, err := a.auditMgr.List(ctx, query)
+	if err != nil {
+		return a.SendError(ctx, err)
+	}
+
+	var auditLogs []*models.AuditLog
+	for _, log := range logs {
+		auditLogs = append(auditLogs, &models.AuditLog{
+			ID:           log.ID,
+			Resource:     log.Resource,
+			ResourceType: log.ResourceType,
+			Username:     log.Username,
+			Operation:    log.Operation,
+			OpTime:       log.OpTime.String(),
+		})
+	}
+	return operation.NewListAuditLogsOK().
+		WithXTotalCount(total).
+		WithLink(a.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
+		WithPayload(auditLogs)
+}

src/server/v2.0/handler/auditlog_test.go

@@ -0,0 +1,3 @@
+package handler
+
+// ToDo add api tests

src/server/v2.0/handler/handler.go

@@ -26,6 +26,7 @@ func New() http.Handler {
 	h, api, err := restapi.HandlerAPI(restapi.Config{
 		ArtifactAPI:   newArtifactAPI(),
 		RepositoryAPI: newRepositoryAPI(),
+		AuditlogAPI:   newAuditLogAPI(),
 	})
 	if err != nil {
 		log.Fatal(err)