Merge pull request #3336 from ywk253100/170927_pro_policy

Implement the default project metadata manager

make/common/db/registry.sql

@@ -97,6 +97,20 @@ create table project_member (
 insert into project_member (project_id, user_id, role, creation_time, update_time) values
 (1, 1, 1, NOW(), NOW());
 
+create table project_metadata (
+ project_id int NOT NULL,
+ name varchar(255) NOT NULL,
+ value varchar(255),
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
+ deleted tinyint (1) DEFAULT 0 NOT NULL,
+ PRIMARY KEY (project_id, name),
+ FOREIGN KEY (project_id) REFERENCES project(project_id)
+);
+
+insert into project_metadata (project_id, name, value, creation_time, update_time, deleted) values
+(1, 'public', 'true', NOW(), NOW(), 0);
+
 create table access_log (
  log_id int NOT NULL AUTO_INCREMENT,
  username varchar (255) NOT NULL,

make/common/db/registry_sqlite.sql

@@ -94,6 +94,20 @@ create table project_member (
 insert into project_member (project_id, user_id, role, creation_time, update_time) values
 (1, 1, 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP);
 
+create table project_metadata (
+ project_id int NOT NULL,
+ name varchar(255) NOT NULL,
+ value varchar(255),
+ creation_time timestamp,
+ update_time timestamp,
+ deleted tinyint (1) DEFAULT 0 NOT NULL,
+ PRIMARY KEY (project_id, name),
+ FOREIGN KEY (project_id) REFERENCES project(project_id)
+);
+
+insert into project_metadata (project_id, name, value, creation_time, update_time, deleted) values
+(1, 'public', 'true', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0);
+
 create table access_log (
  log_id INTEGER PRIMARY KEY,
  username varchar (255) NOT NULL,

src/common/dao/pro_meta.go

@@ -0,0 +1,93 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/vmware/harbor/src/common/models"
+)
+
+// Using raw sql instead of CRUD objects as beego does not support composite primary key
+
+// AddProjectMetadata adds metadata for a project
+func AddProjectMetadata(meta *models.ProjectMetadata) error {
+	now := time.Now()
+	sql := `insert into project_metadata
+				(project_id, name, value, creation_time, update_time, deleted)
+				 values (?, ?, ?, ?, ?, 0)`
+	_, err := GetOrmer().Raw(sql, meta.ProjectID, meta.Name, meta.Value,
+		now, now).Exec()
+	return err
+}
+
+// DeleteProjectMetadata deleted metadata of a project. If name is absent
+// all metadatas will be deleted, otherwise only the metadatas specified
+// by name will be deleted
+func DeleteProjectMetadata(projectID int64, name ...string) error {
+	params := make([]interface{}, 1)
+	sql := `update project_metadata 
+			set deleted = 1 
+			where project_id = ?`
+	params = append(params, projectID)
+
+	if len(name) > 0 {
+		sql += fmt.Sprintf(` and name in ( %s )`, paramPlaceholder(len(name)))
+		params = append(params, name)
+	}
+
+	_, err := GetOrmer().Raw(sql, params).Exec()
+	return err
+}
+
+// UpdateProjectMetadata updates metadata of a project
+func UpdateProjectMetadata(meta *models.ProjectMetadata) error {
+	sql := `update project_metadata 
+				set value = ?, update_time = ? 
+				where project_id = ? and name = ? and deleted = 0`
+	_, err := GetOrmer().Raw(sql, meta.Value, time.Now(), meta.ProjectID,
+		meta.Name).Exec()
+	return err
+}
+
+// GetProjectMetadata returns the metadata of a project. If name is absent
+// all metadatas will be returned, otherwise only the metadatas specified
+// by name will be returned
+func GetProjectMetadata(projectID int64, name ...string) ([]*models.ProjectMetadata, error) {
+	proMetas := []*models.ProjectMetadata{}
+	params := make([]interface{}, 1)
+
+	sql := `select * from project_metadata 
+				where project_id = ? and deleted = 0`
+	params = append(params, projectID)
+
+	if len(name) > 0 {
+		sql += fmt.Sprintf(` and name in ( %s )`, paramPlaceholder(len(name)))
+		params = append(params, name)
+	}
+
+	_, err := GetOrmer().Raw(sql, params).QueryRows(&proMetas)
+	return proMetas, err
+}
+
+func paramPlaceholder(n int) string {
+	placeholders := []string{}
+	for i := 0; i < n; i++ {
+		placeholders = append(placeholders, "?")
+	}
+	return strings.Join(placeholders, ",")
+}

src/common/dao/pro_meta_test.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/vmware/harbor/src/common/models"
+)
+
+func TestProMetaDaoMethods(t *testing.T) {
+	name1 := "key1"
+	value1 := "value1"
+	name2 := "key2"
+	value2 := "value2"
+	meta1 := &models.ProjectMetadata{
+		ProjectID: 1,
+		Name:      name1,
+		Value:     value1,
+	}
+	meta2 := &models.ProjectMetadata{
+		ProjectID: 1,
+		Name:      name2,
+		Value:     value2,
+	}
+	// test add
+	require.Nil(t, AddProjectMetadata(meta1))
+	defer func() {
+		// clean up
+		_, err := GetOrmer().Raw(`delete from project_metadata
+			where project_id = 1 and name = ?`, name1).Exec()
+		require.Nil(t, err)
+	}()
+	require.Nil(t, AddProjectMetadata(meta2))
+	defer func() {
+		// clean up
+		_, err := GetOrmer().Raw(`delete from project_metadata
+			where project_id = 1 and name = ?`, name2).Exec()
+		require.Nil(t, err)
+	}()
+	// test get
+	metas, err := GetProjectMetadata(1, name1, name2)
+	require.Nil(t, err)
+	assert.Equal(t, 2, len(metas))
+
+	m := map[string]*models.ProjectMetadata{}
+	for _, meta := range metas {
+		m[meta.Name] = meta
+	}
+	assert.Equal(t, value1, m[name1].Value)
+	assert.Equal(t, value2, m[name2].Value)
+
+	// test update
+	newValue1 := "new_value1"
+	meta1.Value = newValue1
+	require.Nil(t, UpdateProjectMetadata(meta1))
+	metas, err = GetProjectMetadata(1, name1, name2)
+	require.Nil(t, err)
+	assert.Equal(t, 2, len(metas))
+
+	m = map[string]*models.ProjectMetadata{}
+	for _, meta := range metas {
+		m[meta.Name] = meta
+	}
+	assert.Equal(t, newValue1, m[name1].Value)
+	assert.Equal(t, value2, m[name2].Value)
+
+	// test delete
+	require.Nil(t, DeleteProjectMetadata(1, name1))
+	metas, err = GetProjectMetadata(1, name1, name2)
+	require.Nil(t, err)
+	assert.Equal(t, 1, len(metas))
+	assert.Equal(t, value2, metas[0].Value)
+}

src/common/models/pro_meta.go

@@ -29,7 +29,6 @@ const (
 
 // ProjectMetadata holds the metadata of a project.
 type ProjectMetadata struct {
-	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
 	ProjectID    int64     `orm:"column(project_id)" json:"project_id"`
 	Name         string    `orm:"column(name)" json:"name"`
 	Value        string    `orm:"column(value)" json:"value"`

src/common/models/project.go

@@ -21,18 +21,18 @@ import (
 // Project holds the details of a project.
 // TODO remove useless attrs
 type Project struct {
-	ProjectID       int64                  `orm:"pk;auto;column(project_id)" json:"project_id"`
+	ProjectID       int64             `orm:"pk;auto;column(project_id)" json:"project_id"`
-	OwnerID         int                    `orm:"column(owner_id)" json:"owner_id"`
+	OwnerID         int               `orm:"column(owner_id)" json:"owner_id"`
-	Name            string                 `orm:"column(name)" json:"name"`
+	Name            string            `orm:"column(name)" json:"name"`
-	CreationTime    time.Time              `orm:"column(creation_time)" json:"creation_time"`
+	CreationTime    time.Time         `orm:"column(creation_time)" json:"creation_time"`
-	UpdateTime      time.Time              `orm:"update_time" json:"update_time"`
+	UpdateTime      time.Time         `orm:"update_time" json:"update_time"`
-	Deleted         int                    `orm:"column(deleted)" json:"deleted"`
+	Deleted         int               `orm:"column(deleted)" json:"deleted"`
-	CreationTimeStr string                 `orm:"-" json:"creation_time_str"`
+	CreationTimeStr string            `orm:"-" json:"creation_time_str"`
-	OwnerName       string                 `orm:"-" json:"owner_name"`
+	OwnerName       string            `orm:"-" json:"owner_name"`
-	Togglable       bool                   `orm:"-"`
+	Togglable       bool              `orm:"-"`
-	Role            int                    `orm:"-" json:"current_user_role_id"`
+	Role            int               `orm:"-" json:"current_user_role_id"`
-	RepoCount       int                    `orm:"-" json:"repo_count"`
+	RepoCount       int               `orm:"-" json:"repo_count"`
-	Metadata        map[string]interface{} `orm:"-" json:"metadata"`
+	Metadata        map[string]string `orm:"-" json:"metadata"`
 
 	// TODO remove
 	Public                                     int    `orm:"column(public)" json:"public"`

src/common/utils/test/database.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"os"
+	"strconv"
+
+	"github.com/vmware/harbor/src/common/dao"
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils/log"
+)
+
+// InitDatabaseFromEnv is used to initialize database for testing
+func InitDatabaseFromEnv() {
+	dbHost := os.Getenv("MYSQL_HOST")
+	if len(dbHost) == 0 {
+		log.Fatalf("environment variable MYSQL_HOST is not set")
+	}
+	dbPortStr := os.Getenv("MYSQL_PORT")
+	if len(dbPortStr) == 0 {
+		log.Fatalf("environment variable MYSQL_PORT is not set")
+	}
+	dbPort, err := strconv.Atoi(dbPortStr)
+	if err != nil {
+		log.Fatalf("invalid MYSQL_PORT: %v", err)
+	}
+	dbUser := os.Getenv("MYSQL_USR")
+	if len(dbUser) == 0 {
+		log.Fatalf("environment variable MYSQL_USR is not set")
+	}
+
+	dbPassword := os.Getenv("MYSQL_PWD")
+	dbDatabase := os.Getenv("MYSQL_DATABASE")
+	if len(dbDatabase) == 0 {
+		log.Fatalf("environment variable MYSQL_DATABASE is not set")
+	}
+
+	database := &models.Database{
+		Type: "mysql",
+		MySQL: &models.MySQL{
+			Host:     dbHost,
+			Port:     dbPort,
+			Username: dbUser,
+			Password: dbPassword,
+			Database: dbDatabase,
+		},
+	}
+
+	log.Infof("MYSQL_HOST: %s, MYSQL_USR: %s, MYSQL_PORT: %d, MYSQL_PWD: %s\n", dbHost, dbUser, dbPort, dbPassword)
+
+	if err := dao.InitDatabase(database); err != nil {
+		log.Fatalf("failed to initialize database: %v", err)
+	}
+}

src/ui/api/project.go

@@ -379,8 +379,8 @@ func (p *ProjectAPI) ToggleProjectPublic() {
 
 	if err := p.ProjectMgr.Update(p.project.ProjectID,
 		&models.Project{
-			Metadata: map[string]interface{}{
+			Metadata: map[string]string{
-				models.ProMetaPublic: req.Public,
+				models.ProMetaPublic: strconv.Itoa(req.Public),
 			},
 		}); err != nil {
 		p.ParseAndHandleError(fmt.Sprintf("failed to update project %d",

src/ui/promgr/metamgr/metamgr.go

@@ -15,6 +15,8 @@
 package metamgr
 
 import (
+	"strconv"
+
 	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
 )
@@ -23,15 +25,15 @@ import (
 // implement
 type ProjectMetadataManaegr interface {
 	// Add metadatas for project specified by projectID
-	Add(projectID int64, meta map[string]interface{}) error
+	Add(projectID int64, meta map[string]string) error
 	// Delete metadatas whose keys are specified in parameter meta, if it
 	// is absent, delete all
-	Delete(projecdtID int64, meta ...[]string) error
+	Delete(projecdtID int64, meta ...string) error
 	// Update metadatas
-	Update(projectID int64, meta map[string]interface{}) error
+	Update(projectID int64, meta map[string]string) error
 	// Get metadatas whose keys are specified in parameter meta, if it is
 	// absent, get all
-	Get(projectID int64, meta ...[]string) (map[string]interface{}, error)
+	Get(projectID int64, meta ...string) (map[string]string, error)
 }
 
 type defaultProjectMetadataManaegr struct{}
@@ -41,25 +43,58 @@ func NewDefaultProjectMetadataManager() ProjectMetadataManaegr {
 	return &defaultProjectMetadataManaegr{}
 }
 
-// TODO add implement
+func (d *defaultProjectMetadataManaegr) Add(projectID int64, meta map[string]string) error {
-func (d *defaultProjectMetadataManaegr) Add(projectID int64, meta map[string]interface{}) error {
+	for k, v := range meta {
+		proMeta := &models.ProjectMetadata{
+			ProjectID: projectID,
+			Name:      k,
+			Value:     v,
+		}
+		if err := dao.AddProjectMetadata(proMeta); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
-func (d *defaultProjectMetadataManaegr) Delete(projectID int64, meta ...[]string) error {
+func (d *defaultProjectMetadataManaegr) Delete(projectID int64, meta ...string) error {
-	return nil
+	return dao.DeleteProjectMetadata(projectID, meta...)
 }
 
-func (d *defaultProjectMetadataManaegr) Update(projectID int64, meta map[string]interface{}) error {
+func (d *defaultProjectMetadataManaegr) Update(projectID int64, meta map[string]string) error {
+	for k, v := range meta {
+		if err := dao.UpdateProjectMetadata(&models.ProjectMetadata{
+			ProjectID: projectID,
+			Name:      k,
+			Value:     v,
+		}); err != nil {
+			return err
+		}
+	}
+
 	// TODO remove the logic
 	public, ok := meta[models.ProMetaPublic]
 	if ok {
-		return dao.ToggleProjectPublicity(projectID, public.(int))
+		i, err := strconv.Atoi(public)
+		if err != nil {
+			return err
+		}
+		return dao.ToggleProjectPublicity(projectID, i)
 	}
 
 	return nil
 }
 
-func (d *defaultProjectMetadataManaegr) Get(projectID int64, meta ...[]string) (map[string]interface{}, error) {
+func (d *defaultProjectMetadataManaegr) Get(projectID int64, meta ...string) (map[string]string, error) {
-	return nil, nil
+	proMetas, err := dao.GetProjectMetadata(projectID, meta...)
+	if err != nil {
+		return nil, nil
+	}
+
+	m := map[string]string{}
+	for _, proMeta := range proMetas {
+		m[proMeta.Name] = proMeta.Value
+	}
+
+	return m, nil
 }

src/ui/promgr/metamgr/metamgr_test.go

@@ -14,4 +14,58 @@
 
 package metamgr
 
-// TODO add test cases
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/vmware/harbor/src/common/dao"
+	"github.com/vmware/harbor/src/common/utils/test"
+)
+
+var mgr = NewDefaultProjectMetadataManager()
+
+func TestMain(m *testing.M) {
+	test.InitDatabaseFromEnv()
+	os.Exit(m.Run())
+}
+
+func TestMetaMgrMethods(t *testing.T) {
+	key := "key"
+	value := "value"
+	newValue := "new_value"
+
+	// test add
+	require.Nil(t, mgr.Add(1, map[string]string{
+		key: value,
+	}))
+
+	defer func() {
+		// clean up
+		_, err := dao.GetOrmer().Raw(`delete from project_metadata
+		where project_id = 1 and name = ?`, key).Exec()
+		require.Nil(t, err)
+	}()
+
+	// test get
+	m, err := mgr.Get(1, key)
+	require.Nil(t, err)
+	assert.Equal(t, 1, len(m))
+	assert.Equal(t, value, m[key])
+
+	// test update
+	require.Nil(t, mgr.Update(1, map[string]string{
+		key: newValue,
+	}))
+	m, err = mgr.Get(1, key)
+	require.Nil(t, err)
+	assert.Equal(t, 1, len(m))
+	assert.Equal(t, newValue, m[key])
+
+	// test delete
+	require.Nil(t, mgr.Delete(1, key))
+	m, err = mgr.Get(1, key)
+	require.Nil(t, err)
+	assert.Equal(t, 0, len(m))
+}

src/ui/promgr/promgr.go

@@ -71,7 +71,7 @@ func (d *defaultProjectManager) Get(projectIDOrName interface{}) (*models.Projec
 			return nil, err
 		}
 		if len(project.Metadata) == 0 {
-			project.Metadata = make(map[string]interface{})
+			project.Metadata = make(map[string]string)
 		}
 		for k, v := range meta {
 			project.Metadata[k] = v

src/ui/promgr/promgr_test.go

@@ -87,8 +87,8 @@ func TestDelete(t *testing.T) {
 func TestUpdate(t *testing.T) {
 	assert.Nil(t, proMgr.Update(1,
 		&models.Project{
-			Metadata: map[string]interface{}{
+			Metadata: map[string]string{
-				models.ProMetaPublic: true,
+				models.ProMetaPublic: "true",
 			},
 		}))
 }

tools/migration/changelog.md

@@ -48,4 +48,9 @@ Changelog for harbor database schema
   - alter column `realname` on table `user`: varchar(20)->varchar(255)
   - create table `img_scan_job`
   - create table `img_scan_overview`
-  - create table `clair_vuln_timestamp`
+  - create table `clair_vuln_timestamp`
+
+## 1.3.0
+
+  - create table `project_metadata`
+  - insert data into table `project_metadata`