From 15e4361d6e6ec912daff77689b12a7b04dcdc8db Mon Sep 17 00:00:00 2001 From: chlins Date: Mon, 29 Jun 2020 13:16:29 +0800 Subject: [PATCH] feat: add p2p preheat policy dao and manager(#12286) Signed-off-by: chlins --- .../postgresql/0040_2.1.0_schema.up.sql | 15 +- src/pkg/p2p/preheat/dao/policy/dao.go | 147 ++++++++++++++++ src/pkg/p2p/preheat/dao/policy/dao_test.go | 158 ++++++++++++++++++ src/pkg/p2p/preheat/models/policy/policy.go | 18 +- .../p2p/preheat/models/policy/policy_test.go | 1 - src/pkg/p2p/preheat/policy/manager.go | 93 +++++++++++ src/pkg/p2p/preheat/policy/manager_test.go | 123 ++++++++++++++ 7 files changed, 549 insertions(+), 6 deletions(-) create mode 100644 src/pkg/p2p/preheat/dao/policy/dao.go create mode 100644 src/pkg/p2p/preheat/dao/policy/dao_test.go create mode 100644 src/pkg/p2p/preheat/policy/manager.go create mode 100644 src/pkg/p2p/preheat/policy/manager_test.go diff --git a/make/migrations/postgresql/0040_2.1.0_schema.up.sql b/make/migrations/postgresql/0040_2.1.0_schema.up.sql index 21703e6f8..7e66e5138 100644 --- a/make/migrations/postgresql/0040_2.1.0_schema.up.sql +++ b/make/migrations/postgresql/0040_2.1.0_schema.up.sql @@ -35,4 +35,17 @@ ALTER TABLE blob ADD COLUMN IF NOT EXISTS update_time timestamp default CURRENT_ ALTER TABLE blob ADD COLUMN IF NOT EXISTS status varchar(255); ALTER TABLE blob ADD COLUMN IF NOT EXISTS version BIGINT default 0; CREATE INDEX IF NOT EXISTS idx_status ON blob (status); -CREATE INDEX IF NOT EXISTS idx_version ON blob (version); \ No newline at end of file +CREATE INDEX IF NOT EXISTS idx_version ON blob (version); + +CREATE TABLE IF NOT EXISTS p2p_preheat_policy ( + id SERIAL PRIMARY KEY NOT NULL, + name varchar(255) NOT NULL, + description varchar(1024), + project_id int NOT NULL, + provider_id int NOT NULL, + filters varchar(1024), + trigger varchar(16), + enabled boolean, + creation_time timestamp, + update_time timestamp +); diff --git a/src/pkg/p2p/preheat/dao/policy/dao.go b/src/pkg/p2p/preheat/dao/policy/dao.go new file mode 100644 index 000000000..cf3a13bb2 --- /dev/null +++ b/src/pkg/p2p/preheat/dao/policy/dao.go @@ -0,0 +1,147 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy + +import ( + "context" + + beego_orm "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/policy" +) + +// DAO is the data access object for policy. +type DAO interface { + // Create the policy schema + Create(ctx context.Context, schema *policy.Schema) (id int64, err error) + // Update the policy schema, Only the properties specified by "props" will be updated if it is set + Update(ctx context.Context, schema *policy.Schema, props ...string) (err error) + // Get the policy schema by id + Get(ctx context.Context, id int64) (schema *policy.Schema, err error) + // Delete the policy schema by id + Delete(ctx context.Context, id int64) (err error) + // List policy schemas by query + List(ctx context.Context, query *q.Query) (total int64, schemas []*policy.Schema, err error) +} + +// New returns an instance of the default DAO. +func New() DAO { + return &dao{} +} + +type dao struct{} + +// Create a policy schema. +func (d *dao) Create(ctx context.Context, schema *policy.Schema) (id int64, err error) { + var ormer beego_orm.Ormer + ormer, err = orm.FromContext(ctx) + if err != nil { + return + } + + id, err = ormer.Insert(schema) + if err != nil { + if e := orm.AsConflictError(err, "policy %s already exists", schema.Name); e != nil { + err = e + } + return + } + + return +} + +// Update a policy schema. +func (d *dao) Update(ctx context.Context, schema *policy.Schema, props ...string) (err error) { + var ormer beego_orm.Ormer + ormer, err = orm.FromContext(ctx) + if err != nil { + return err + } + + id, err := ormer.Update(schema, props...) + if err != nil { + return err + } + + if id == 0 { + return errors.NotFoundError(nil).WithMessage("policy %d not found", schema.ID) + } + + return nil +} + +// Get a policy schema by id. +func (d *dao) Get(ctx context.Context, id int64) (schema *policy.Schema, err error) { + var ormer beego_orm.Ormer + ormer, err = orm.FromContext(ctx) + if err != nil { + return + } + + schema = &policy.Schema{ID: id} + if err = ormer.Read(schema); err != nil { + if e := orm.AsNotFoundError(err, "policy %d not found", id); e != nil { + err = e + } + return nil, err + } + + return schema, nil +} + +// Delete a policy schema by id. +func (d *dao) Delete(ctx context.Context, id int64) (err error) { + var ormer beego_orm.Ormer + ormer, err = orm.FromContext(ctx) + if err != nil { + return + } + + n, err := ormer.Delete(&policy.Schema{ + ID: id, + }) + if err != nil { + return err + } + + if n == 0 { + return errors.NotFoundError(nil).WithMessage("policy %d not found", id) + } + + return nil +} + +// List policies by query. +func (d *dao) List(ctx context.Context, query *q.Query) (total int64, schemas []*policy.Schema, err error) { + var qs beego_orm.QuerySeter + qs, err = orm.QuerySetter(ctx, &policy.Schema{}, query) + if err != nil { + return + } + + total, err = qs.Count() + if err != nil { + return + } + + qs = qs.OrderBy("UpdatedTime", "ID") + if _, err = qs.All(&schemas); err != nil { + return + } + + return total, schemas, nil +} diff --git a/src/pkg/p2p/preheat/dao/policy/dao_test.go b/src/pkg/p2p/preheat/dao/policy/dao_test.go new file mode 100644 index 000000000..cfbf65c2f --- /dev/null +++ b/src/pkg/p2p/preheat/dao/policy/dao_test.go @@ -0,0 +1,158 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy + +import ( + "context" + "testing" + "time" + + beego_orm "github.com/astaxie/beego/orm" + common_dao "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/lib/orm" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/policy" + "github.com/stretchr/testify/suite" +) + +type daoTestSuite struct { + suite.Suite + + dao DAO + ctx context.Context + defaultPolicy *policy.Schema +} + +// TestDaoTestSuite tests policy dao. +func TestDaoTestSuite(t *testing.T) { + suite.Run(t, &daoTestSuite{}) +} + +// SetupSuite setups testing env. +func (d *daoTestSuite) SetupSuite() { + common_dao.PrepareTestForPostgresSQL() + d.dao = New() + d.ctx = orm.NewContext(nil, beego_orm.NewOrm()) + d.defaultPolicy = &policy.Schema{ + ID: 1, + Name: "default-policy", + Description: "test", + ProjectID: 1, + ProviderID: 1, + Filters: nil, + FiltersStr: "", + Trigger: nil, + TriggerStr: "", + Enabled: true, + CreatedAt: time.Now(), + UpdatedTime: time.Now(), + } + + _, err := d.dao.Create(d.ctx, d.defaultPolicy) + d.Require().Nil(err) +} + +// TearDownTest cleans testing env. +func (d *daoTestSuite) TearDownSuite() { + err := d.dao.Delete(d.ctx, d.defaultPolicy.ID) + d.Require().Nil(err) +} + +// TestCreate tests create a policy schema. +func (d *daoTestSuite) TestCreate() { + // create duplicate policy should return error + _, err := d.dao.Create(d.ctx, d.defaultPolicy) + d.Require().NotNil(err) + d.True(errors.IsErr(err, errors.ConflictCode)) +} + +// Delete tests delete a policy schema. +func (d *daoTestSuite) TestDelete() { + // delete a not exist policy + err := d.dao.Delete(d.ctx, 0) + d.Require().NotNil(err) + d.True(errors.IsErr(err, errors.NotFoundCode)) +} + +// Get tests get a policy schema by id. +func (d *daoTestSuite) TestGet() { + policy, err := d.dao.Get(d.ctx, 1) + d.Require().Nil(err) + d.Require().NotNil(policy) + d.Equal(d.defaultPolicy.Name, policy.Name, "get a default policy") + + // not found + _, err = d.dao.Get(d.ctx, 1000) + d.Require().NotNil(err) + d.True(errors.IsErr(err, errors.NotFoundCode)) +} + +// Update tests update a policy schema. +func (d *daoTestSuite) TestUpdate() { + newDesc := "test update" + newPolicy := *d.defaultPolicy + newPolicy.Description = newDesc + + err := d.dao.Update(d.ctx, &newPolicy) + d.Require().Nil(err) + + policy, err := d.dao.Get(d.ctx, 1) + d.Require().Nil(err) + d.Require().NotNil(policy) + d.Equal(newDesc, policy.Description, "update a policy description") +} + +func (d *daoTestSuite) TestList() { + newPolicy := &policy.Schema{ + ID: 2, + Name: "new-policy", + Description: "new", + ProjectID: 2, + ProviderID: 2, + Filters: nil, + FiltersStr: "", + Trigger: nil, + TriggerStr: "", + Enabled: false, + CreatedAt: time.Time{}, + UpdatedTime: time.Time{}, + } + + _, err := d.dao.Create(d.ctx, newPolicy) + d.Require().Nil(err) + // clean up + defer func() { + err = d.dao.Delete(d.ctx, 2) + d.Require().Nil(err) + }() + + total, policies, err := d.dao.List(d.ctx, &q.Query{}) + d.Require().Nil(err) + d.Equal(int64(2), total) + d.Len(policies, 2, "list all policy schemas") + + // list policy filter by project + query := &q.Query{ + Keywords: map[string]interface{}{ + "project_id": 1, + }, + } + total, policies, err = d.dao.List(d.ctx, query) + d.Require().Nil(err) + d.Equal(int64(1), total) + d.Len(policies, 1, "list policy schemas by project") + d.Equal(d.defaultPolicy.Name, policies[0].Name) +} diff --git a/src/pkg/p2p/preheat/models/policy/policy.go b/src/pkg/p2p/preheat/models/policy/policy.go index af6102320..cb9a74dc0 100644 --- a/src/pkg/p2p/preheat/models/policy/policy.go +++ b/src/pkg/p2p/preheat/models/policy/policy.go @@ -18,10 +18,15 @@ import ( "fmt" "time" + beego_orm "github.com/astaxie/beego/orm" "github.com/astaxie/beego/validation" "github.com/robfig/cron" ) +func init() { + beego_orm.RegisterModel(&Schema{}) +} + const ( // Filters: // Repository : type=Repository value=name text (double star pattern used) @@ -54,13 +59,13 @@ type Schema struct { ID int64 `orm:"column(id)" json:"id"` Name string `orm:"column(name)" json:"name"` Description string `orm:"column(description)" json:"description"` - // use project name - Project string `orm:"column(project)" json:"project"` + // use project id + ProjectID int64 `orm:"column(project_id)" json:"project_id"` ProviderID int64 `orm:"column(provider_id)" json:"provider_id"` - Filters []*Filter `orm:"column(-)" json:"filters"` + Filters []*Filter `orm:"-" json:"filters"` // Use JSON data format （query by filter type should be supported) FiltersStr string `orm:"column(filters)" json:"-"` - Trigger *Trigger `orm:"column(-)" json:"trigger"` + Trigger *Trigger `orm:"-" json:"trigger"` // Use JSON data format (query by trigger type should be supported) TriggerStr string `orm:"column(trigger)" json:"-"` Enabled bool `orm:"column(enabled)" json:"enabled"` @@ -68,6 +73,11 @@ type Schema struct { UpdatedTime time.Time `orm:"column(update_time)" json:"update_time"` } +// TableName specifies the policy schema table name. +func (s *Schema) TableName() string { + return "p2p_preheat_policy" +} + // FilterType represents the type info of the filter. type FilterType = string diff --git a/src/pkg/p2p/preheat/models/policy/policy_test.go b/src/pkg/p2p/preheat/models/policy/policy_test.go index aa2ebb45b..cf1aff195 100644 --- a/src/pkg/p2p/preheat/models/policy/policy_test.go +++ b/src/pkg/p2p/preheat/models/policy/policy_test.go @@ -19,7 +19,6 @@ import ( "github.com/astaxie/beego/validation" "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" ) diff --git a/src/pkg/p2p/preheat/policy/manager.go b/src/pkg/p2p/preheat/policy/manager.go new file mode 100644 index 000000000..e9754eea9 --- /dev/null +++ b/src/pkg/p2p/preheat/policy/manager.go @@ -0,0 +1,93 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy + +import ( + "context" + + "github.com/goharbor/harbor/src/lib/q" + dao "github.com/goharbor/harbor/src/pkg/p2p/preheat/dao/policy" + "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/policy" +) + +// Mgr is a global instance of policy manager +var Mgr = New() + +// Manager manages the policy +type Manager interface { + // Create the policy schema + Create(ctx context.Context, schema *policy.Schema) (id int64, err error) + // Update the policy schema, Only the properties specified by "props" will be updated if it is set + Update(ctx context.Context, schema *policy.Schema, props ...string) (err error) + // Get the policy schema by id + Get(ctx context.Context, id int64) (schema *policy.Schema, err error) + // Delete the policy schema by id + Delete(ctx context.Context, id int64) (err error) + // List policy schemas by query + ListPolicies(ctx context.Context, query *q.Query) (total int64, schemas []*policy.Schema, err error) + // list policy schema under project + ListPoliciesByProject(ctx context.Context, project int64, query *q.Query) (total int64, schemas []*policy.Schema, err error) +} + +type manager struct { + dao dao.DAO +} + +// New creates an instance of the default policy manager +func New() Manager { + return &manager{ + dao: dao.New(), + } +} + +// Create the policy schema +func (m *manager) Create(ctx context.Context, schema *policy.Schema) (id int64, err error) { + return m.dao.Create(ctx, schema) +} + +// Update the policy schema, Only the properties specified by "props" will be updated if it is set +func (m *manager) Update(ctx context.Context, schema *policy.Schema, props ...string) (err error) { + return m.dao.Update(ctx, schema, props...) +} + +// Get the policy schema by id +func (m *manager) Get(ctx context.Context, id int64) (schema *policy.Schema, err error) { + return m.dao.Get(ctx, id) +} + +// Delete the policy schema by id +func (m *manager) Delete(ctx context.Context, id int64) (err error) { + return m.dao.Delete(ctx, id) +} + +// List policy schemas by query +func (m *manager) ListPolicies(ctx context.Context, query *q.Query) (total int64, schemas []*policy.Schema, err error) { + return m.dao.List(ctx, query) +} + +// list policy schema under project +func (m *manager) ListPoliciesByProject(ctx context.Context, project int64, query *q.Query) (total int64, schemas []*policy.Schema, err error) { + if query == nil { + query = &q.Query{} + } + + if query.Keywords == nil { + query.Keywords = make(map[string]interface{}) + } + // set project filter + query.Keywords["project_id"] = project + + return m.dao.List(ctx, query) +} diff --git a/src/pkg/p2p/preheat/policy/manager_test.go b/src/pkg/p2p/preheat/policy/manager_test.go new file mode 100644 index 000000000..056e27122 --- /dev/null +++ b/src/pkg/p2p/preheat/policy/manager_test.go @@ -0,0 +1,123 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy + +import ( + "context" + "testing" + + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/p2p/preheat/models/policy" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type fakeDao struct { + mock.Mock +} + +func (f *fakeDao) Create(ctx context.Context, schema *policy.Schema) (int64, error) { + args := f.Called() + return int64(args.Int(0)), args.Error(1) +} +func (f *fakeDao) Update(ctx context.Context, schema *policy.Schema, props ...string) error { + args := f.Called() + return args.Error(0) +} +func (f *fakeDao) Get(ctx context.Context, id int64) (*policy.Schema, error) { + args := f.Called() + var schema *policy.Schema + if args.Get(0) != nil { + schema = args.Get(0).(*policy.Schema) + } + return schema, args.Error(1) +} +func (f *fakeDao) Delete(ctx context.Context, id int64) error { + args := f.Called() + return args.Error(0) +} +func (f *fakeDao) List(ctx context.Context, query *q.Query) (int64, []*policy.Schema, error) { + args := f.Called() + var schemas []*policy.Schema + if args.Get(0) != nil { + schemas = args.Get(0).([]*policy.Schema) + } + return 0, schemas, args.Error(1) +} + +type managerTestSuite struct { + suite.Suite + mgr Manager + dao *fakeDao +} + +// TestManagerTestSuite tests managerTestSuite +func TestManagerTestSuite(t *testing.T) { + suite.Run(t, &managerTestSuite{}) +} + +// SetupSuite setups testing env. +func (m *managerTestSuite) SetupSuite() { + m.dao = &fakeDao{} + m.mgr = &manager{dao: m.dao} +} + +// TearDownSuite cleans testing env. +func (m *managerTestSuite) TearDownSuite() { + m.dao = nil + m.mgr = nil +} + +// TestCreate tests Create method. +func (m *managerTestSuite) TestCreate() { + m.dao.On("Create").Return(1, nil) + _, err := m.mgr.Create(nil, nil) + m.Require().Nil(err) +} + +// TestUpdate tests Update method. +func (m *managerTestSuite) TestUpdate() { + m.dao.On("Update").Return(nil) + err := m.mgr.Update(nil, nil) + m.Require().Nil(err) +} + +// TestGet tests Get method. +func (m *managerTestSuite) TestGet() { + m.dao.On("Get").Return(nil, nil) + _, err := m.mgr.Get(nil, 1) + m.Require().Nil(err) +} + +// TestDelete tests Delete method. +func (m *managerTestSuite) TestDelete() { + m.dao.On("Delete").Return(nil) + err := m.mgr.Delete(nil, 1) + m.Require().Nil(err) +} + +// TestListPolicies tests ListPolicies method. +func (m *managerTestSuite) TestListPolicies() { + m.dao.On("List").Return(nil, nil) + _, _, err := m.mgr.ListPolicies(nil, nil) + m.Require().Nil(err) +} + +// TestListPoliciesByProject tests ListPoliciesByProject method. +func (m *managerTestSuite) TestListPoliciesByProject() { + m.dao.On("List").Return(nil, nil) + _, _, err := m.mgr.ListPoliciesByProject(nil, 1, nil) + m.Require().Nil(err) +}