add audit logs API

1, add API entry for get audit logs
2. add audit log manager to hanlder CRUD

Use the new format of audit log to cover differernt resource, artifact/tag/repostory/project

Signed-off-by: wang yan <wangyan@vmware.com>

api/v2.0/swagger.yaml

@@ -499,6 +499,68 @@ paths:
           $ref: '#/responses/409'
         '500':
           $ref: '#/responses/500'
+  /audit-logs:
+    get:
+      summary: Get recent logs of the projects which the user is a member of
+      description: |
+        This endpoint let user see the recent operation logs of the projects which he is member of
+      tags:
+        - auditlog
+      operationId: listAuditLogs
+      parameters:
+        - $ref: '#/parameters/requestId'
+        - $ref: '#/parameters/page'
+        - $ref: '#/parameters/pageSize'
+        - name: username
+          in: query
+          type: string
+          required: false
+          description: Username of the operator.
+        - name: resource
+          in: query
+          type: string
+          required: false
+          description: The identity of resource
+        - name: resource_type
+          in: query
+          type: string
+          required: false
+          description: The type of resource, artifact/tag/repository
+        - name: operation
+          in: query
+          type: string
+          required: false
+          description: The operation, create/delete
+        - name: begin_timestamp
+          in: query
+          type: string
+          required: false
+          description: The begin timestamp
+        - name: end_timestamp
+          in: query
+          type: string
+          required: false
+          description: The end timestamp
+      responses:
+        '200':
+          description: Success
+          headers:
+            X-Total-Count:
+              description: The total count of auditlogs
+              type: integer
+            Link:
+              description: Link refers to the previous page and next page
+              type: string
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/AuditLog'
+        '400':
+          $ref: '#/responses/400'
+        '401':
+          $ref: '#/responses/401'
+        '500':
+          $ref: '#/responses/500'
 parameters:
   requestId:
     name: X-Request-Id
@@ -919,3 +981,24 @@ definitions:
         example:
           'Critical': 5
           'High': 5
+  AuditLog:
+    type: object
+    properties:
+      id:
+        type: integer
+        description: The ID of the audit log entry.
+      username:
+        type: string
+        description: Username of the user in this log entry.
+      resource:
+        type: string
+        description: Name of the repository in this log entry.
+      resource_type:
+        type: string
+        description: Tag of the repository in this log entry.
+      operation:
+        type: string
+        description: The operation against the repository in this log entry.
+      op_time:
+        type: string
+        description: The time when this operation is triggered.

make/migrations/postgresql/0030_2.0.0_schema.up.sql

@@ -173,3 +173,14 @@ CREATE TABLE artifact_2
   push_time     timestamp,
   CONSTRAINT unique_artifact_2 UNIQUE (project_id, repo, tag)
 );
+
+CREATE TABLE audit_log
+(
+ id             SERIAL PRIMARY KEY NOT NULL,
+ project_id     int NOT NULL,
+ operation      varchar(20) NOT NULL,
+ resource_type  varchar(255) NOT NULL,
+ resource       varchar(1024) NOT NULL,
+ username       varchar(255) NOT NULL,
+ op_time        timestamp default CURRENT_TIMESTAMP
+);

src/pkg/audit/dao/dao.go

@@ -0,0 +1,127 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"context"
+	ierror "github.com/goharbor/harbor/src/internal/error"
+	"github.com/goharbor/harbor/src/internal/orm"
+	"github.com/goharbor/harbor/src/pkg/audit/model"
+	"github.com/goharbor/harbor/src/pkg/q"
+)
+
+// DAO is the data access object for audit log
+type DAO interface {
+	// Create the audit log
+	Create(ctx context.Context, access *model.AuditLog) (id int64, err error)
+	// Count returns the total count of audit logs according to the query
+	Count(ctx context.Context, query *q.Query) (total int64, err error)
+	// List audit logs according to the query
+	List(ctx context.Context, query *q.Query) (access []*model.AuditLog, err error)
+	// Get the audit log specified by ID
+	Get(ctx context.Context, id int64) (access *model.AuditLog, err error)
+	// Delete the audit log specified by ID
+	Delete(ctx context.Context, id int64) (err error)
+}
+
+// New returns an instance of the default DAO
+func New() DAO {
+	return &dao{}
+}
+
+type dao struct{}
+
+// Count ...
+func (d *dao) Count(ctx context.Context, query *q.Query) (int64, error) {
+	if query != nil {
+		// ignore the page number and size
+		query = &q.Query{
+			Keywords: query.Keywords,
+		}
+	}
+	qs, err := orm.QuerySetter(ctx, &model.AuditLog{}, query)
+	if err != nil {
+		return 0, err
+	}
+	return qs.Count()
+}
+
+// List ...
+func (d *dao) List(ctx context.Context, query *q.Query) ([]*model.AuditLog, error) {
+	audit := []*model.AuditLog{}
+	qs, err := orm.QuerySetter(ctx, &model.AuditLog{}, query)
+	qs = qs.OrderBy("-op_time")
+	if err != nil {
+		return nil, err
+	}
+	if _, err = qs.All(&audit); err != nil {
+		return nil, err
+	}
+	return audit, nil
+}
+
+// Get ...
+func (d *dao) Get(ctx context.Context, id int64) (*model.AuditLog, error) {
+	audit := &model.AuditLog{
+		ID: id,
+	}
+	ormer, err := orm.FromContext(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if err := ormer.Read(audit); err != nil {
+		if e := orm.AsNotFoundError(err, "audit %d not found", id); e != nil {
+			err = e
+		}
+		return nil, err
+	}
+	return audit, nil
+}
+
+// Create ...
+func (d *dao) Create(ctx context.Context, audit *model.AuditLog) (int64, error) {
+	ormer, err := orm.FromContext(ctx)
+	if err != nil {
+		return 0, err
+	}
+	// the max length of username in database is 255, replace the last
+	// three charaters with "..." if the length is greater than 256
+	if len(audit.Username) > 255 {
+		audit.Username = audit.Username[:252] + "..."
+	}
+	id, err := ormer.Insert(audit)
+	if err != nil {
+		return 0, err
+	}
+	return id, err
+}
+
+// Delete ...
+func (d *dao) Delete(ctx context.Context, id int64) error {
+	ormer, err := orm.FromContext(ctx)
+	if err != nil {
+		return err
+	}
+	n, err := ormer.Delete(&model.AuditLog{
+		ID: id,
+	})
+	if err != nil {
+		return err
+	}
+	if n == 0 {
+		return ierror.NotFoundError(nil).WithMessage("access %d not found", id)
+	}
+	return nil
+}

src/pkg/audit/dao/dao_test.go

@@ -0,0 +1,119 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"context"
+	"errors"
+	beegoorm "github.com/astaxie/beego/orm"
+	common_dao "github.com/goharbor/harbor/src/common/dao"
+	ierror "github.com/goharbor/harbor/src/internal/error"
+	"github.com/goharbor/harbor/src/internal/orm"
+	"github.com/goharbor/harbor/src/pkg/audit/model"
+	"github.com/goharbor/harbor/src/pkg/q"
+	"github.com/stretchr/testify/suite"
+	"testing"
+)
+
+type daoTestSuite struct {
+	suite.Suite
+	dao     DAO
+	auditID int64
+	ctx     context.Context
+}
+
+func (d *daoTestSuite) SetupSuite() {
+	d.dao = New()
+	common_dao.PrepareTestForPostgresSQL()
+	d.ctx = orm.NewContext(nil, beegoorm.NewOrm())
+	artifactID, err := d.dao.Create(d.ctx, &model.AuditLog{
+		Operation:    "Create",
+		ResourceType: "artifact",
+		Resource:     "library/hello-world",
+		Username:     "admin",
+	})
+	d.Require().Nil(err)
+	d.auditID = artifactID
+}
+
+func (d *daoTestSuite) TearDownSuite() {
+	err := d.dao.Delete(d.ctx, d.auditID)
+	d.Require().Nil(err)
+}
+
+func (d *daoTestSuite) TestCount() {
+	total, err := d.dao.Count(d.ctx, nil)
+	d.Require().Nil(err)
+	d.True(total > 0)
+	total, err = d.dao.Count(d.ctx, &q.Query{
+		Keywords: map[string]interface{}{
+			"ResourceType": "artifact",
+		},
+	})
+	d.Require().Nil(err)
+	d.Equal(int64(1), total)
+}
+
+func (d *daoTestSuite) TestList() {
+	// nil query
+	audits, err := d.dao.List(d.ctx, nil)
+	d.Require().Nil(err)
+
+	// query by repository ID and name
+	audits, err = d.dao.List(d.ctx, &q.Query{
+		Keywords: map[string]interface{}{
+			"ResourceType": "artifact",
+		},
+	})
+	d.Require().Nil(err)
+	d.Require().Equal(1, len(audits))
+	d.Equal("admin", audits[0].Username)
+}
+
+func (d *daoTestSuite) TestGet() {
+	// get the non-exist tag
+	_, err := d.dao.Get(d.ctx, 10000)
+	d.Require().NotNil(err)
+	d.True(ierror.IsErr(err, ierror.NotFoundCode))
+
+	audit, err := d.dao.Get(d.ctx, d.auditID)
+	d.Require().Nil(err)
+	d.Require().NotNil(audit)
+	d.Equal(d.auditID, audit.ID)
+}
+
+func (d *daoTestSuite) TestCreate() {
+	// conflict
+	audit := &model.AuditLog{
+		Operation:    "Create",
+		ResourceType: "tag",
+		Resource:     "library/hello-world",
+		Username:     "admin",
+	}
+	_, err := d.dao.Create(d.ctx, audit)
+	d.Require().Nil(err)
+}
+
+func (d *daoTestSuite) TestDelete() {
+	err := d.dao.Delete(d.ctx, 10000)
+	d.Require().NotNil(err)
+	var e *ierror.Error
+	d.Require().True(errors.As(err, &e))
+	d.Equal(ierror.NotFoundCode, e.Code)
+}
+
+func TestDaoTestSuite(t *testing.T) {
+	suite.Run(t, &daoTestSuite{})
+}

src/pkg/audit/manager.go

@@ -0,0 +1,75 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package audit
+
+import (
+	"context"
+	"github.com/goharbor/harbor/src/pkg/audit/dao"
+	"github.com/goharbor/harbor/src/pkg/audit/model"
+	"github.com/goharbor/harbor/src/pkg/q"
+)
+
+// Mgr is the global audit log manager instance
+var Mgr = New()
+
+// Manager is used for audit log management
+type Manager interface {
+	// Count returns the total count of audit logs according to the query
+	Count(ctx context.Context, query *q.Query) (total int64, err error)
+	// List audit logs according to the query
+	List(ctx context.Context, query *q.Query) (audits []*model.AuditLog, err error)
+	// Get the audit log specified by ID
+	Get(ctx context.Context, id int64) (audit *model.AuditLog, err error)
+	// Create the audit log
+	Create(ctx context.Context, audit *model.AuditLog) (id int64, err error)
+	// Delete the audit log specified by ID
+	Delete(ctx context.Context, id int64) (err error)
+}
+
+// New returns a default implementation of Manager
+func New() Manager {
+	return &manager{
+		dao: dao.New(),
+	}
+}
+
+type manager struct {
+	dao dao.DAO
+}
+
+// Count ...
+func (m *manager) Count(ctx context.Context, query *q.Query) (int64, error) {
+	return m.dao.Count(ctx, query)
+}
+
+// List ...
+func (m *manager) List(ctx context.Context, query *q.Query) ([]*model.AuditLog, error) {
+	return m.dao.List(ctx, query)
+}
+
+// Get ...
+func (m *manager) Get(ctx context.Context, id int64) (*model.AuditLog, error) {
+	return m.dao.Get(ctx, id)
+}
+
+// Create ...
+func (m *manager) Create(ctx context.Context, audit *model.AuditLog) (int64, error) {
+	return m.dao.Create(ctx, audit)
+}
+
+// Delete ...
+func (m *manager) Delete(ctx context.Context, id int64) error {
+	return m.dao.Delete(ctx, id)
+}

src/pkg/audit/manager_test.go

@@ -0,0 +1,119 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package audit
+
+import (
+	"context"
+	"github.com/goharbor/harbor/src/pkg/audit/model"
+	"github.com/goharbor/harbor/src/pkg/q"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/suite"
+	"testing"
+)
+
+type fakeDao struct {
+	mock.Mock
+}
+
+func (f *fakeDao) Count(ctx context.Context, query *q.Query) (int64, error) {
+	args := f.Called()
+	return int64(args.Int(0)), args.Error(1)
+}
+func (f *fakeDao) List(ctx context.Context, query *q.Query) ([]*model.AuditLog, error) {
+	args := f.Called()
+	return args.Get(0).([]*model.AuditLog), args.Error(1)
+}
+func (f *fakeDao) Get(ctx context.Context, id int64) (*model.AuditLog, error) {
+	args := f.Called()
+	return args.Get(0).(*model.AuditLog), args.Error(1)
+}
+func (f *fakeDao) Create(ctx context.Context, repository *model.AuditLog) (int64, error) {
+	args := f.Called()
+	return int64(args.Int(0)), args.Error(1)
+}
+func (f *fakeDao) Delete(ctx context.Context, id int64) error {
+	args := f.Called()
+	return args.Error(0)
+}
+
+type managerTestSuite struct {
+	suite.Suite
+	mgr *manager
+	dao *fakeDao
+}
+
+func (m *managerTestSuite) SetupTest() {
+	m.dao = &fakeDao{}
+	m.mgr = &manager{
+		dao: m.dao,
+	}
+}
+
+func (m *managerTestSuite) TestCount() {
+	m.dao.On("Count", mock.Anything).Return(1, nil)
+	total, err := m.mgr.Count(nil, nil)
+	m.Require().Nil(err)
+	m.Equal(int64(1), total)
+}
+
+func (m *managerTestSuite) TestList() {
+	audit := &model.AuditLog{
+		ProjectID:    1,
+		Resource:     "library/hello-world",
+		ResourceType: "artifact",
+	}
+	m.dao.On("List", mock.Anything).Return([]*model.AuditLog{audit}, nil)
+	auditLogs, err := m.mgr.List(nil, nil)
+	m.Require().Nil(err)
+	m.Equal(1, len(auditLogs))
+	m.Equal(audit.Resource, auditLogs[0].Resource)
+}
+
+func (m *managerTestSuite) TestGet() {
+	audit := &model.AuditLog{
+		ProjectID:    1,
+		Resource:     "library/hello-world",
+		ResourceType: "artifact",
+	}
+	m.dao.On("Get", mock.Anything).Return(audit, nil)
+	au, err := m.mgr.Get(nil, 1)
+	m.Require().Nil(err)
+	m.dao.AssertExpectations(m.T())
+	m.Require().NotNil(au)
+	m.Equal(audit.Resource, au.Resource)
+}
+
+func (m *managerTestSuite) TestCreate() {
+	m.dao.On("Create", mock.Anything).Return(1, nil)
+	id, err := m.mgr.Create(nil, &model.AuditLog{
+		ProjectID:    1,
+		Resource:     "library/hello-world",
+		ResourceType: "artifact",
+	})
+	m.Require().Nil(err)
+	m.dao.AssertExpectations(m.T())
+	m.Equal(int64(1), id)
+}
+
+func (m *managerTestSuite) TestDelete() {
+	m.dao.On("Delete", mock.Anything).Return(nil)
+	err := m.mgr.Delete(nil, 1)
+	m.Require().Nil(err)
+	m.dao.AssertExpectations(m.T())
+}
+
+func TestManager(t *testing.T) {
+	suite.Run(t, &managerTestSuite{})
+}

src/pkg/audit/model/model.go

@@ -0,0 +1,26 @@
+package model
+
+import (
+	beego_orm "github.com/astaxie/beego/orm"
+	"time"
+)
+
+func init() {
+	beego_orm.RegisterModel(&AuditLog{})
+}
+
+// AuditLog ...
+type AuditLog struct {
+	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
+	ProjectID    int       `orm:"column(project_id)" json:"project_id"`
+	Operation    string    `orm:"column(operation)" json:"operation"`
+	ResourceType string    `orm:"column(resource_type)"  json:"resource_type"`
+	Resource     string    `orm:"column(resource)" json:"resource"`
+	Username     string    `orm:"column(username)"  json:"username"`
+	OpTime       time.Time `orm:"column(op_time)" json:"op_time"`
+}
+
+// TableName for audit log
+func (a *AuditLog) TableName() string {
+	return "audit_log"
+}

src/server/v2.0/handler/auditlog.go

@@ -0,0 +1,76 @@
+package handler
+
+import (
+	"context"
+	"github.com/go-openapi/runtime/middleware"
+	"github.com/goharbor/harbor/src/pkg/audit"
+	"github.com/goharbor/harbor/src/pkg/q"
+	"github.com/goharbor/harbor/src/server/v2.0/models"
+	"github.com/goharbor/harbor/src/server/v2.0/restapi/operations/auditlog"
+	operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/auditlog"
+)
+
+func newAuditLogAPI() *auditlogAPI {
+	return &auditlogAPI{
+		auditMgr: audit.Mgr,
+	}
+}
+
+type auditlogAPI struct {
+	BaseAPI
+	auditMgr audit.Manager
+}
+
+func (a *auditlogAPI) ListAuditLogs(ctx context.Context, params auditlog.ListAuditLogsParams) middleware.Responder {
+	// ToDo enable permission check
+	// if !a.HasPermission(ctx, rbac.ActionList, rbac.ResourceLog) {
+	//	return a.SendError(ctx, ierror.ForbiddenError(nil))
+	// }
+	keywords := make(map[string]interface{})
+	query := &q.Query{
+		Keywords: keywords,
+	}
+	// TODO support fuzzy match and start end time
+	if params.Username != nil {
+		query.Keywords["Username"] = *(params.Username)
+	}
+	if params.Operation != nil {
+		query.Keywords["Operation"] = *(params.Operation)
+	}
+	if params.Resource != nil {
+		query.Keywords["Resource"] = *(params.Resource)
+	}
+	if params.ResourceType != nil {
+		query.Keywords["ResourceType"] = *(params.ResourceType)
+	}
+	if params.Page != nil {
+		query.PageNumber = *(params.Page)
+	}
+	if params.PageSize != nil {
+		query.PageSize = *(params.PageSize)
+	}
+	total, err := a.auditMgr.Count(ctx, query)
+	if err != nil {
+		return a.SendError(ctx, err)
+	}
+	logs, err := a.auditMgr.List(ctx, query)
+	if err != nil {
+		return a.SendError(ctx, err)
+	}
+
+	var auditLogs []*models.AuditLog
+	for _, log := range logs {
+		auditLogs = append(auditLogs, &models.AuditLog{
+			ID:           log.ID,
+			Resource:     log.Resource,
+			ResourceType: log.ResourceType,
+			Username:     log.Username,
+			Operation:    log.Operation,
+			OpTime:       log.OpTime.String(),
+		})
+	}
+	return operation.NewListAuditLogsOK().
+		WithXTotalCount(total).
+		WithLink(a.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String()).
+		WithPayload(auditLogs)
+}

src/server/v2.0/handler/auditlog_test.go

@@ -0,0 +1,3 @@
+package handler
+
+// ToDo add api tests

src/server/v2.0/handler/handler.go

@@ -26,6 +26,7 @@ func New() http.Handler {
 	h, api, err := restapi.HandlerAPI(restapi.Config{
 		ArtifactAPI:   newArtifactAPI(),
 		RepositoryAPI: newRepositoryAPI(),
+		AuditlogAPI:   newAuditLogAPI(),
 	})
 	if err != nil {
 		log.Fatal(err)