Merge pull request #12342 from chlins/feat/p2p-preheat-policy-dao-and-manager

feat: add p2p preheat policy dao and manager(#12286)

make/migrations/postgresql/0040_2.1.0_schema.up.sql

@@ -35,4 +35,17 @@ ALTER TABLE blob ADD COLUMN IF NOT EXISTS update_time timestamp default CURRENT_
 ALTER TABLE blob ADD COLUMN IF NOT EXISTS status varchar(255);
 ALTER TABLE blob ADD COLUMN IF NOT EXISTS version BIGINT default 0;
 CREATE INDEX IF NOT EXISTS idx_status ON blob (status);
-CREATE INDEX IF NOT EXISTS idx_version ON blob (version);
+CREATE INDEX IF NOT EXISTS idx_version ON blob (version);
+
+CREATE TABLE IF NOT EXISTS p2p_preheat_policy (
+    id SERIAL PRIMARY KEY NOT NULL,
+    name varchar(255) NOT NULL,
+    description varchar(1024),
+    project_id int NOT NULL,
+    provider_id int NOT NULL,
+    filters varchar(1024),
+    trigger varchar(16),
+    enabled boolean,
+    creation_time timestamp,
+    update_time timestamp
+);

src/pkg/p2p/preheat/dao/policy/dao.go

@@ -0,0 +1,147 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package policy
+
+import (
+	"context"
+
+	beego_orm "github.com/astaxie/beego/orm"
+	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/lib/orm"
+	"github.com/goharbor/harbor/src/lib/q"
+	"github.com/goharbor/harbor/src/pkg/p2p/preheat/models/policy"
+)
+
+// DAO is the data access object for policy.
+type DAO interface {
+	// Create the policy schema
+	Create(ctx context.Context, schema *policy.Schema) (id int64, err error)
+	// Update the policy schema, Only the properties specified by "props" will be updated if it is set
+	Update(ctx context.Context, schema *policy.Schema, props ...string) (err error)
+	// Get the policy schema by id
+	Get(ctx context.Context, id int64) (schema *policy.Schema, err error)
+	// Delete the policy schema by id
+	Delete(ctx context.Context, id int64) (err error)
+	// List policy schemas by query
+	List(ctx context.Context, query *q.Query) (total int64, schemas []*policy.Schema, err error)
+}
+
+// New returns an instance of the default DAO.
+func New() DAO {
+	return &dao{}
+}
+
+type dao struct{}
+
+// Create a policy schema.
+func (d *dao) Create(ctx context.Context, schema *policy.Schema) (id int64, err error) {
+	var ormer beego_orm.Ormer
+	ormer, err = orm.FromContext(ctx)
+	if err != nil {
+		return
+	}
+
+	id, err = ormer.Insert(schema)
+	if err != nil {
+		if e := orm.AsConflictError(err, "policy %s already exists", schema.Name); e != nil {
+			err = e
+		}
+		return
+	}
+
+	return
+}
+
+// Update a policy schema.
+func (d *dao) Update(ctx context.Context, schema *policy.Schema, props ...string) (err error) {
+	var ormer beego_orm.Ormer
+	ormer, err = orm.FromContext(ctx)
+	if err != nil {
+		return err
+	}
+
+	id, err := ormer.Update(schema, props...)
+	if err != nil {
+		return err
+	}
+
+	if id == 0 {
+		return errors.NotFoundError(nil).WithMessage("policy %d not found", schema.ID)
+	}
+
+	return nil
+}
+
+// Get a policy schema by id.
+func (d *dao) Get(ctx context.Context, id int64) (schema *policy.Schema, err error) {
+	var ormer beego_orm.Ormer
+	ormer, err = orm.FromContext(ctx)
+	if err != nil {
+		return
+	}
+
+	schema = &policy.Schema{ID: id}
+	if err = ormer.Read(schema); err != nil {
+		if e := orm.AsNotFoundError(err, "policy %d not found", id); e != nil {
+			err = e
+		}
+		return nil, err
+	}
+
+	return schema, nil
+}
+
+// Delete a policy schema by id.
+func (d *dao) Delete(ctx context.Context, id int64) (err error) {
+	var ormer beego_orm.Ormer
+	ormer, err = orm.FromContext(ctx)
+	if err != nil {
+		return
+	}
+
+	n, err := ormer.Delete(&policy.Schema{
+		ID: id,
+	})
+	if err != nil {
+		return err
+	}
+
+	if n == 0 {
+		return errors.NotFoundError(nil).WithMessage("policy %d not found", id)
+	}
+
+	return nil
+}
+
+// List policies by query.
+func (d *dao) List(ctx context.Context, query *q.Query) (total int64, schemas []*policy.Schema, err error) {
+	var qs beego_orm.QuerySeter
+	qs, err = orm.QuerySetter(ctx, &policy.Schema{}, query)
+	if err != nil {
+		return
+	}
+
+	total, err = qs.Count()
+	if err != nil {
+		return
+	}
+
+	qs = qs.OrderBy("UpdatedTime", "ID")
+	if _, err = qs.All(&schemas); err != nil {
+		return
+	}
+
+	return total, schemas, nil
+}

src/pkg/p2p/preheat/dao/policy/dao_test.go

@@ -0,0 +1,158 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package policy
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	beego_orm "github.com/astaxie/beego/orm"
+	common_dao "github.com/goharbor/harbor/src/common/dao"
+	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/lib/orm"
+	"github.com/goharbor/harbor/src/lib/q"
+	"github.com/goharbor/harbor/src/pkg/p2p/preheat/models/policy"
+	"github.com/stretchr/testify/suite"
+)
+
+type daoTestSuite struct {
+	suite.Suite
+
+	dao           DAO
+	ctx           context.Context
+	defaultPolicy *policy.Schema
+}
+
+// TestDaoTestSuite tests policy dao.
+func TestDaoTestSuite(t *testing.T) {
+	suite.Run(t, &daoTestSuite{})
+}
+
+// SetupSuite setups testing env.
+func (d *daoTestSuite) SetupSuite() {
+	common_dao.PrepareTestForPostgresSQL()
+	d.dao = New()
+	d.ctx = orm.NewContext(nil, beego_orm.NewOrm())
+	d.defaultPolicy = &policy.Schema{
+		ID:          1,
+		Name:        "default-policy",
+		Description: "test",
+		ProjectID:   1,
+		ProviderID:  1,
+		Filters:     nil,
+		FiltersStr:  "",
+		Trigger:     nil,
+		TriggerStr:  "",
+		Enabled:     true,
+		CreatedAt:   time.Now(),
+		UpdatedTime: time.Now(),
+	}
+
+	_, err := d.dao.Create(d.ctx, d.defaultPolicy)
+	d.Require().Nil(err)
+}
+
+// TearDownTest cleans testing env.
+func (d *daoTestSuite) TearDownSuite() {
+	err := d.dao.Delete(d.ctx, d.defaultPolicy.ID)
+	d.Require().Nil(err)
+}
+
+// TestCreate tests create a policy schema.
+func (d *daoTestSuite) TestCreate() {
+	// create duplicate policy should return error
+	_, err := d.dao.Create(d.ctx, d.defaultPolicy)
+	d.Require().NotNil(err)
+	d.True(errors.IsErr(err, errors.ConflictCode))
+}
+
+// Delete tests delete a policy schema.
+func (d *daoTestSuite) TestDelete() {
+	// delete a not exist policy
+	err := d.dao.Delete(d.ctx, 0)
+	d.Require().NotNil(err)
+	d.True(errors.IsErr(err, errors.NotFoundCode))
+}
+
+// Get tests get a policy schema by id.
+func (d *daoTestSuite) TestGet() {
+	policy, err := d.dao.Get(d.ctx, 1)
+	d.Require().Nil(err)
+	d.Require().NotNil(policy)
+	d.Equal(d.defaultPolicy.Name, policy.Name, "get a default policy")
+
+	// not found
+	_, err = d.dao.Get(d.ctx, 1000)
+	d.Require().NotNil(err)
+	d.True(errors.IsErr(err, errors.NotFoundCode))
+}
+
+// Update tests update a policy schema.
+func (d *daoTestSuite) TestUpdate() {
+	newDesc := "test update"
+	newPolicy := *d.defaultPolicy
+	newPolicy.Description = newDesc
+
+	err := d.dao.Update(d.ctx, &newPolicy)
+	d.Require().Nil(err)
+
+	policy, err := d.dao.Get(d.ctx, 1)
+	d.Require().Nil(err)
+	d.Require().NotNil(policy)
+	d.Equal(newDesc, policy.Description, "update a policy description")
+}
+
+func (d *daoTestSuite) TestList() {
+	newPolicy := &policy.Schema{
+		ID:          2,
+		Name:        "new-policy",
+		Description: "new",
+		ProjectID:   2,
+		ProviderID:  2,
+		Filters:     nil,
+		FiltersStr:  "",
+		Trigger:     nil,
+		TriggerStr:  "",
+		Enabled:     false,
+		CreatedAt:   time.Time{},
+		UpdatedTime: time.Time{},
+	}
+
+	_, err := d.dao.Create(d.ctx, newPolicy)
+	d.Require().Nil(err)
+	// clean up
+	defer func() {
+		err = d.dao.Delete(d.ctx, 2)
+		d.Require().Nil(err)
+	}()
+
+	total, policies, err := d.dao.List(d.ctx, &q.Query{})
+	d.Require().Nil(err)
+	d.Equal(int64(2), total)
+	d.Len(policies, 2, "list all policy schemas")
+
+	// list policy filter by project
+	query := &q.Query{
+		Keywords: map[string]interface{}{
+			"project_id": 1,
+		},
+	}
+	total, policies, err = d.dao.List(d.ctx, query)
+	d.Require().Nil(err)
+	d.Equal(int64(1), total)
+	d.Len(policies, 1, "list policy schemas by project")
+	d.Equal(d.defaultPolicy.Name, policies[0].Name)
+}

src/pkg/p2p/preheat/models/policy/policy.go

@@ -18,10 +18,15 @@ import (
 	"fmt"
 	"time"
 
+	beego_orm "github.com/astaxie/beego/orm"
 	"github.com/astaxie/beego/validation"
 	"github.com/robfig/cron"
 )
 
+func init() {
+	beego_orm.RegisterModel(&Schema{})
+}
+
 const (
 	// Filters:
 	//  Repository : type=Repository value=name text (double star pattern used)
@@ -54,13 +59,13 @@ type Schema struct {
 	ID          int64  `orm:"column(id)" json:"id"`
 	Name        string `orm:"column(name)" json:"name"`
 	Description string `orm:"column(description)" json:"description"`
-	// use project name
+	// use project id
-	Project    string    `orm:"column(project)" json:"project"`
+	ProjectID  int64     `orm:"column(project_id)" json:"project_id"`
 	ProviderID int64     `orm:"column(provider_id)" json:"provider_id"`
-	Filters    []*Filter `orm:"column(-)" json:"filters"`
+	Filters    []*Filter `orm:"-" json:"filters"`
 	// Use JSON data format （query by filter type should be supported)
 	FiltersStr string   `orm:"column(filters)" json:"-"`
-	Trigger    *Trigger `orm:"column(-)" json:"trigger"`
+	Trigger    *Trigger `orm:"-" json:"trigger"`
 	// Use JSON data format (query by trigger type should be supported)
 	TriggerStr  string    `orm:"column(trigger)" json:"-"`
 	Enabled     bool      `orm:"column(enabled)" json:"enabled"`
@@ -68,6 +73,11 @@ type Schema struct {
 	UpdatedTime time.Time `orm:"column(update_time)" json:"update_time"`
 }
 
+// TableName specifies the policy schema table name.
+func (s *Schema) TableName() string {
+	return "p2p_preheat_policy"
+}
+
 // FilterType represents the type info of the filter.
 type FilterType = string
 

src/pkg/p2p/preheat/models/policy/policy_test.go

@@ -19,7 +19,6 @@ import (
 
 	"github.com/astaxie/beego/validation"
 	"github.com/stretchr/testify/require"
-
 	"github.com/stretchr/testify/suite"
 )
 

src/pkg/p2p/preheat/policy/manager.go

@@ -0,0 +1,93 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package policy
+
+import (
+	"context"
+
+	"github.com/goharbor/harbor/src/lib/q"
+	dao "github.com/goharbor/harbor/src/pkg/p2p/preheat/dao/policy"
+	"github.com/goharbor/harbor/src/pkg/p2p/preheat/models/policy"
+)
+
+// Mgr is a global instance of policy manager
+var Mgr = New()
+
+// Manager manages the policy
+type Manager interface {
+	// Create the policy schema
+	Create(ctx context.Context, schema *policy.Schema) (id int64, err error)
+	// Update the policy schema, Only the properties specified by "props" will be updated if it is set
+	Update(ctx context.Context, schema *policy.Schema, props ...string) (err error)
+	// Get the policy schema by id
+	Get(ctx context.Context, id int64) (schema *policy.Schema, err error)
+	// Delete the policy schema by id
+	Delete(ctx context.Context, id int64) (err error)
+	// List policy schemas by query
+	ListPolicies(ctx context.Context, query *q.Query) (total int64, schemas []*policy.Schema, err error)
+	// list policy schema under project
+	ListPoliciesByProject(ctx context.Context, project int64, query *q.Query) (total int64, schemas []*policy.Schema, err error)
+}
+
+type manager struct {
+	dao dao.DAO
+}
+
+// New creates an instance of the default policy manager
+func New() Manager {
+	return &manager{
+		dao: dao.New(),
+	}
+}
+
+// Create the policy schema
+func (m *manager) Create(ctx context.Context, schema *policy.Schema) (id int64, err error) {
+	return m.dao.Create(ctx, schema)
+}
+
+// Update the policy schema, Only the properties specified by "props" will be updated if it is set
+func (m *manager) Update(ctx context.Context, schema *policy.Schema, props ...string) (err error) {
+	return m.dao.Update(ctx, schema, props...)
+}
+
+// Get the policy schema by id
+func (m *manager) Get(ctx context.Context, id int64) (schema *policy.Schema, err error) {
+	return m.dao.Get(ctx, id)
+}
+
+// Delete the policy schema by id
+func (m *manager) Delete(ctx context.Context, id int64) (err error) {
+	return m.dao.Delete(ctx, id)
+}
+
+// List policy schemas by query
+func (m *manager) ListPolicies(ctx context.Context, query *q.Query) (total int64, schemas []*policy.Schema, err error) {
+	return m.dao.List(ctx, query)
+}
+
+// list policy schema under project
+func (m *manager) ListPoliciesByProject(ctx context.Context, project int64, query *q.Query) (total int64, schemas []*policy.Schema, err error) {
+	if query == nil {
+		query = &q.Query{}
+	}
+
+	if query.Keywords == nil {
+		query.Keywords = make(map[string]interface{})
+	}
+	// set project filter
+	query.Keywords["project_id"] = project
+
+	return m.dao.List(ctx, query)
+}

src/pkg/p2p/preheat/policy/manager_test.go

@@ -0,0 +1,123 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package policy
+
+import (
+	"context"
+	"testing"
+
+	"github.com/goharbor/harbor/src/lib/q"
+	"github.com/goharbor/harbor/src/pkg/p2p/preheat/models/policy"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/suite"
+)
+
+type fakeDao struct {
+	mock.Mock
+}
+
+func (f *fakeDao) Create(ctx context.Context, schema *policy.Schema) (int64, error) {
+	args := f.Called()
+	return int64(args.Int(0)), args.Error(1)
+}
+func (f *fakeDao) Update(ctx context.Context, schema *policy.Schema, props ...string) error {
+	args := f.Called()
+	return args.Error(0)
+}
+func (f *fakeDao) Get(ctx context.Context, id int64) (*policy.Schema, error) {
+	args := f.Called()
+	var schema *policy.Schema
+	if args.Get(0) != nil {
+		schema = args.Get(0).(*policy.Schema)
+	}
+	return schema, args.Error(1)
+}
+func (f *fakeDao) Delete(ctx context.Context, id int64) error {
+	args := f.Called()
+	return args.Error(0)
+}
+func (f *fakeDao) List(ctx context.Context, query *q.Query) (int64, []*policy.Schema, error) {
+	args := f.Called()
+	var schemas []*policy.Schema
+	if args.Get(0) != nil {
+		schemas = args.Get(0).([]*policy.Schema)
+	}
+	return 0, schemas, args.Error(1)
+}
+
+type managerTestSuite struct {
+	suite.Suite
+	mgr Manager
+	dao *fakeDao
+}
+
+// TestManagerTestSuite tests managerTestSuite
+func TestManagerTestSuite(t *testing.T) {
+	suite.Run(t, &managerTestSuite{})
+}
+
+// SetupSuite setups testing env.
+func (m *managerTestSuite) SetupSuite() {
+	m.dao = &fakeDao{}
+	m.mgr = &manager{dao: m.dao}
+}
+
+// TearDownSuite cleans testing env.
+func (m *managerTestSuite) TearDownSuite() {
+	m.dao = nil
+	m.mgr = nil
+}
+
+// TestCreate tests Create method.
+func (m *managerTestSuite) TestCreate() {
+	m.dao.On("Create").Return(1, nil)
+	_, err := m.mgr.Create(nil, nil)
+	m.Require().Nil(err)
+}
+
+// TestUpdate tests Update method.
+func (m *managerTestSuite) TestUpdate() {
+	m.dao.On("Update").Return(nil)
+	err := m.mgr.Update(nil, nil)
+	m.Require().Nil(err)
+}
+
+// TestGet tests Get method.
+func (m *managerTestSuite) TestGet() {
+	m.dao.On("Get").Return(nil, nil)
+	_, err := m.mgr.Get(nil, 1)
+	m.Require().Nil(err)
+}
+
+// TestDelete tests Delete method.
+func (m *managerTestSuite) TestDelete() {
+	m.dao.On("Delete").Return(nil)
+	err := m.mgr.Delete(nil, 1)
+	m.Require().Nil(err)
+}
+
+// TestListPolicies tests ListPolicies method.
+func (m *managerTestSuite) TestListPolicies() {
+	m.dao.On("List").Return(nil, nil)
+	_, _, err := m.mgr.ListPolicies(nil, nil)
+	m.Require().Nil(err)
+}
+
+// TestListPoliciesByProject tests ListPoliciesByProject method.
+func (m *managerTestSuite) TestListPoliciesByProject() {
+	m.dao.On("List").Return(nil, nil)
+	_, _, err := m.mgr.ListPoliciesByProject(nil, 1, nil)
+	m.Require().Nil(err)
+}