Merge pull request #12673 from chlins/fix/preheat-execution-and-task-pagination

fix(preheat): add pagination for execution and task list api

api/v2.0/swagger.yaml

@@ -1030,6 +1030,13 @@ paths:
       responses:
         '200':
           description: List executions success
+          headers:
+            X-Total-Count:
+              description: The total count of tags
+              type: integer
+            Link:
+              description: Link refers to the previous page and next page
+              type: string
           schema:
             type: array
             items:
@@ -1113,9 +1120,19 @@ paths:
         - $ref: '#/parameters/projectName'
         - $ref: '#/parameters/preheatPolicyName'
         - $ref: '#/parameters/executionId'
+        - $ref: '#/parameters/page'
+        - $ref: '#/parameters/pageSize'
+        - $ref: '#/parameters/query'
       responses:
         '200':
           description: List tasks success
+          headers:
+            X-Total-Count:
+              description: The total count of tags
+              type: integer
+            Link:
+              description: Link refers to the previous page and next page
+              type: string
           schema:
             type: array
             items:

src/controller/task/controller.go

@@ -36,6 +36,8 @@ type Controller interface {
 	List(ctx context.Context, query *q.Query) (tasks []*task.Task, err error)
 	// Get the log of the specified task.
 	GetLog(ctx context.Context, id int64) (log []byte, err error)
+	// Count counts total.
+	Count(ctx context.Context, query *q.Query) (int64, error)
 }
 
 // NewController creates an instance of the default task controller.
@@ -50,6 +52,11 @@ type controller struct {
 	mgr task.Manager
 }
 
+// Count counts total.
+func (c *controller) Count(ctx context.Context, query *q.Query) (int64, error) {
+	return c.mgr.Count(ctx, query)
+}
+
 // Stop the specified task.
 func (c *controller) Stop(ctx context.Context, id int64) (err error) {
 	return c.mgr.Stop(ctx, id)

src/controller/task/controller_test.go

@@ -17,6 +17,8 @@ package task
 import (
 	"testing"
 
+	"github.com/goharbor/harbor/src/lib/q"
+
 	model "github.com/goharbor/harbor/src/pkg/task"
 	"github.com/goharbor/harbor/src/testing/mock"
 	"github.com/goharbor/harbor/src/testing/pkg/task"
@@ -40,6 +42,14 @@ func (c *controllerTestSuite) SetupTest() {
 	c.ctl = &controller{mgr: c.mgr}
 }
 
+// TestCount tests count.
+func (c *controllerTestSuite) TestCount() {
+	c.mgr.On("Count", mock.Anything, mock.Anything).Return(int64(10), nil)
+	total, err := c.ctl.Count(nil, &q.Query{})
+	c.NoError(err)
+	c.Equal(int64(10), total)
+}
+
 // TestStop tests stop.
 func (c *controllerTestSuite) TestStop() {
 	c.mgr.On("Stop", mock.Anything, mock.Anything).Return(nil)

src/controller/task/execution_controller.go

@@ -36,6 +36,8 @@ type ExecutionController interface {
 	Get(ctx context.Context, id int64) (execution *task.Execution, err error)
 	// List executions according to the query.
 	List(ctx context.Context, query *q.Query) (executions []*task.Execution, err error)
+	// Count counts total.
+	Count(ctx context.Context, query *q.Query) (int64, error)
 }
 
 // executionController defines the execution controller.
@@ -69,3 +71,8 @@ func (ec *executionController) Get(ctx context.Context, id int64) (execution *ta
 func (ec *executionController) List(ctx context.Context, query *q.Query) (executions []*task.Execution, err error) {
 	return ec.mgr.List(ctx, query)
 }
+
+// Count counts total.
+func (ec *executionController) Count(ctx context.Context, query *q.Query) (int64, error) {
+	return ec.mgr.Count(ctx, query)
+}

src/controller/task/execution_controller_test.go

@@ -17,6 +17,8 @@ package task
 import (
 	"testing"
 
+	"github.com/goharbor/harbor/src/lib/q"
+
 	model "github.com/goharbor/harbor/src/pkg/task"
 	"github.com/goharbor/harbor/src/testing/mock"
 	"github.com/goharbor/harbor/src/testing/pkg/task"
@@ -42,6 +44,14 @@ func (ec *executionControllerTestSuite) SetupTest() {
 	}
 }
 
+// TestCount tests count.
+func (ec *executionControllerTestSuite) TestCount() {
+	ec.mgr.On("Count", mock.Anything, mock.Anything).Return(int64(10), nil)
+	total, err := ec.ctl.Count(nil, &q.Query{})
+	ec.NoError(err)
+	ec.Equal(int64(10), total)
+}
+
 // TestStop tests stop.
 func (ec *executionControllerTestSuite) TestStop() {
 	ec.mgr.On("Stop", mock.Anything, mock.Anything).Return(nil)

src/pkg/task/execution.go

@@ -58,6 +58,8 @@ type ExecutionManager interface {
 	Get(ctx context.Context, id int64) (execution *Execution, err error)
 	// List executions according to the query
 	List(ctx context.Context, query *q.Query) (executions []*Execution, err error)
+	// Count counts total.
+	Count(ctx context.Context, query *q.Query) (int64, error)
 }
 
 // NewExecutionManager return an instance of the default execution manager
@@ -75,6 +77,10 @@ type executionManager struct {
 	taskDAO      dao.TaskDAO
 }
 
+func (e *executionManager) Count(ctx context.Context, query *q.Query) (int64, error) {
+	return e.executionDAO.Count(ctx, query)
+}
+
 func (e *executionManager) Create(ctx context.Context, vendorType string, vendorID int64, trigger string,
 	extraAttrs ...map[string]interface{}) (int64, error) {
 	extras := map[string]interface{}{}

src/pkg/task/execution_test.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/lib/q"
 	"github.com/goharbor/harbor/src/pkg/task/dao"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
@@ -44,6 +45,14 @@ func (e *executionManagerTestSuite) SetupTest() {
 	}
 }
 
+func (e *executionManagerTestSuite) TestCount() {
+	e.execDAO.On("Count", mock.Anything, mock.Anything).Return(int64(10), nil)
+	total, err := e.execMgr.Count(nil, &q.Query{})
+	e.Require().Nil(err)
+	e.Equal(int64(10), total)
+	e.execDAO.AssertExpectations(e.T())
+}
+
 func (e *executionManagerTestSuite) TestCreate() {
 	e.execDAO.On("Create", mock.Anything, mock.Anything).Return(int64(1), nil)
 	id, err := e.execMgr.Create(nil, "vendor", 0, ExecutionTriggerManual,

src/pkg/task/mock_task_manager_test.go

@@ -14,6 +14,27 @@ type mockTaskManager struct {
 	mock.Mock
 }
 
+// Count provides a mock function with given fields: ctx, query
+func (_m *mockTaskManager) Count(ctx context.Context, query *q.Query) (int64, error) {
+	ret := _m.Called(ctx, query)
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok {
+		r0 = rf(ctx, query)
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
+		r1 = rf(ctx, query)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // Create provides a mock function with given fields: ctx, executionID, job, extraAttrs
 func (_m *mockTaskManager) Create(ctx context.Context, executionID int64, job *Job, extraAttrs ...map[string]interface{}) (int64, error) {
 	_va := make([]interface{}, len(extraAttrs))

src/pkg/task/task.go

@@ -50,6 +50,8 @@ type Manager interface {
 	List(ctx context.Context, query *q.Query) (tasks []*Task, err error)
 	// Get the log of the specified task
 	GetLog(ctx context.Context, id int64) (log []byte, err error)
+	// Count counts total.
+	Count(ctx context.Context, query *q.Query) (int64, error)
 }
 
 // NewManager creates an instance of the default task manager
@@ -67,6 +69,10 @@ type manager struct {
 	coreURL  string
 }
 
+func (m *manager) Count(ctx context.Context, query *q.Query) (int64, error) {
+	return m.dao.Count(ctx, query)
+}
+
 func (m *manager) Create(ctx context.Context, executionID int64, jb *Job, extraAttrs ...map[string]interface{}) (int64, error) {
 	// create task record in database
 	id, err := m.createTaskRecord(ctx, executionID, extraAttrs...)

src/pkg/task/task_test.go

@@ -16,9 +16,11 @@ package task
 
 import (
 	"errors"
-	cjob "github.com/goharbor/harbor/src/common/job"
 	"testing"
 
+	cjob "github.com/goharbor/harbor/src/common/job"
+	"github.com/goharbor/harbor/src/lib/q"
+
 	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/pkg/task/dao"
 	"github.com/stretchr/testify/mock"
@@ -41,6 +43,14 @@ func (t *taskManagerTestSuite) SetupTest() {
 	}
 }
 
+func (t *taskManagerTestSuite) TestCount() {
+	t.dao.On("Count", mock.Anything, mock.Anything).Return(int64(10), nil)
+	total, err := t.mgr.Count(nil, &q.Query{})
+	t.Require().Nil(err)
+	t.Equal(int64(10), total)
+	t.dao.AssertExpectations(t.T())
+}
+
 func (t *taskManagerTestSuite) TestCreate() {
 	// success to submit job to jobservice
 	t.dao.On("Create", mock.Anything, mock.Anything).Return(int64(1), nil)

src/server/v2.0/handler/preheat.go

@@ -605,6 +605,11 @@ func (api *preheatAPI) ListExecutions(ctx context.Context, params operation.List
 		query.Keywords["vendor_id"] = policy.ID
 	}
 
+	total, err := api.executionCtl.Count(ctx, query)
+	if err != nil {
+		return api.SendError(ctx, err)
+	}
+
 	executions, err := api.executionCtl.List(ctx, query)
 	if err != nil {
 		return api.SendError(ctx, err)
@@ -619,7 +624,8 @@ func (api *preheatAPI) ListExecutions(ctx context.Context, params operation.List
 		payloads = append(payloads, p)
 	}
 
-	return operation.NewListExecutionsOK().WithPayload(payloads)
+	return operation.NewListExecutionsOK().WithPayload(payloads).WithXTotalCount(total).
+		WithLink(api.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String())
 }
 
 // StopExecution stops execution.
@@ -666,10 +672,18 @@ func (api *preheatAPI) ListTasks(ctx context.Context, params operation.ListTasks
 		return api.SendError(ctx, err)
 	}
 
-	query := &q.Query{
+	query, err := api.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
-		Keywords: map[string]interface{}{
+	if err != nil {
-			"execution_id": params.ExecutionID,
+		return api.SendError(ctx, err)
-		},
+	}
+
+	if query != nil {
+		query.Keywords["execution_id"] = params.ExecutionID
+	}
+
+	total, err := api.taskCtl.Count(ctx, query)
+	if err != nil {
+		return api.SendError(ctx, err)
 	}
 
 	tasks, err := api.taskCtl.List(ctx, query)
@@ -686,7 +700,8 @@ func (api *preheatAPI) ListTasks(ctx context.Context, params operation.ListTasks
 		payloads = append(payloads, p)
 	}
 
-	return operation.NewListTasksOK().WithPayload(payloads)
+	return operation.NewListTasksOK().WithPayload(payloads).WithXTotalCount(total).
+		WithLink(api.Links(ctx, params.HTTPRequest.URL, total, query.PageNumber, query.PageSize).String())
 }
 
 // GetLog gets log.

src/testing/pkg/task/execution_manager.go

@@ -16,6 +16,27 @@ type FakeExecutionManager struct {
 	mock.Mock
 }
 
+// Count provides a mock function with given fields: ctx, query
+func (_m *FakeExecutionManager) Count(ctx context.Context, query *q.Query) (int64, error) {
+	ret := _m.Called(ctx, query)
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok {
+		r0 = rf(ctx, query)
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
+		r1 = rf(ctx, query)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // Create provides a mock function with given fields: ctx, vendorType, vendorID, trigger, extraAttrs
 func (_m *FakeExecutionManager) Create(ctx context.Context, vendorType string, vendorID int64, trigger string, extraAttrs ...map[string]interface{}) (int64, error) {
 	_va := make([]interface{}, len(extraAttrs))

src/testing/pkg/task/manager.go

@@ -16,6 +16,27 @@ type FakeManager struct {
 	mock.Mock
 }
 
+// Count provides a mock function with given fields: ctx, query
+func (_m *FakeManager) Count(ctx context.Context, query *q.Query) (int64, error) {
+	ret := _m.Called(ctx, query)
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func(context.Context, *q.Query) int64); ok {
+		r0 = rf(ctx, query)
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, *q.Query) error); ok {
+		r1 = rf(ctx, query)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // Create provides a mock function with given fields: ctx, executionID, job, extraAttrs
 func (_m *FakeManager) Create(ctx context.Context, executionID int64, job *task.Job, extraAttrs ...map[string]interface{}) (int64, error) {
 	_va := make([]interface{}, len(extraAttrs))