Merge pull request #3640 from yixingjia/moveconftoDB

Add database driver for Harbor configurations

.travis.yml

@@ -97,7 +97,8 @@ script:
   - sudo -E env "PATH=$PATH" ./tests/coverage4gotest.sh
   - goveralls -coverprofile=profile.cov -service=travis-ci
   - docker-compose -f make/docker-compose.test.yml down
-  - sudo rm -rf /data/config/*  
+  - sudo rm -rf /data/config/*
+  - sudo rm -rf /data/database/*
   - ls /data/cert
   - sudo make install GOBUILDIMAGE=golang:1.7.3 COMPILETAG=compile_golangimage CLARITYIMAGE=vmware/harbor-clarity-ui-builder:1.2.7 NOTARYFLAG=true CLAIRFLAG=true
   - sleep 10  

make/common/db/registry.sql

@@ -221,9 +221,11 @@ UNIQUE(namespace)
 );
 
 create table properties (
+ id int NOT NULL AUTO_INCREMENT,
  k varchar(64) NOT NULL,
  v varchar(128) NOT NULL,
- primary key (k)
+ PRIMARY KEY(id),
+ UNIQUE (k)
  );
 
 CREATE TABLE IF NOT EXISTS `alembic_version` (

make/common/db/registry_sqlite.sql

@@ -212,9 +212,10 @@ UNIQUE(namespace)
 );
 
 create table properties (
+ id INTEGER PRIMARY KEY,
  k varchar(64) NOT NULL,
  v varchar(128) NOT NULL,
- primary key (k)
+ UNIQUE(k)
  );
 
 create table alembic_version (

src/adminserver/systemcfg/store/database/driver_db.go

@@ -0,0 +1,122 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package database
+
+import (
+	"github.com/vmware/harbor/src/common/dao"
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/adminserver/systemcfg/store"
+	"github.com/vmware/harbor/src/common"
+	"fmt"
+	"strconv"
+)
+
+const (
+	name = "database"
+	)
+var(
+	numKeys = map[string]bool{
+		common.EmailPort:true,
+		common.LDAPScope:true,
+		common.LDAPTimeout:true,
+		common.TokenExpiration:true,
+		common.MySQLPort:true,
+		common.MaxJobWorkers:true,
+		common.CfgExpiration:true,
+		}
+	boolKeys = map[string]bool{
+		common.WithClair:true,
+		common.WithNotary:true,
+		common.SelfRegistration:true,
+		common.EmailSSL:true,
+		common.EmailInsecure:true,
+		common.LDAPVerifyCert:true,
+	}
+	)
+type cfgStore struct {
+	name string
+}
+
+// Name The name of the driver
+func (c *cfgStore) Name() string {
+	return name
+}
+
+// NewCfgStore New a cfg store for database driver
+func NewCfgStore()  (store.Driver, error){
+	return &cfgStore{
+		name: name,
+	}, nil
+}
+// Read configuration from database
+func (c *cfgStore) Read() (map[string]interface{}, error) {
+	configEntries,error := dao.GetConfigEntries()
+	if error != nil {
+		return nil, error
+	}
+	return WrapperConfig(configEntries)
+}
+
+// WrapperConfig Wrapper the configuration
+func WrapperConfig (configEntries []*models.ConfigEntry) (map[string]interface{}, error) {
+	config := make(map[string]interface{})
+	for _,entry := range configEntries{
+		if numKeys[entry.Key]{
+			strvalue, err := strconv.Atoi(entry.Value)
+			if err != nil {
+				return nil, err
+			}
+			config[entry.Key] = float64(strvalue)
+		}else if boolKeys[entry.Key] {
+			strvalue, err := strconv.ParseBool(entry.Value)
+			if err != nil {
+				return nil, err
+			}
+			config[entry.Key]=strvalue
+		}else{
+			config[entry.Key] = entry.Value
+		}
+
+	}
+	return config, nil
+}
+// Write save configuration to database
+func (c *cfgStore) Write(config map[string]interface{}) error {
+	configEntries ,_:= TranslateConfig(config)
+	return dao.SaveConfigEntries(configEntries)
+}
+
+// TranslateConfig Translate configuration from int, bool, float64 to string
+func TranslateConfig(config map[string]interface{}) ([]models.ConfigEntry,error) {
+	var configEntries []models.ConfigEntry
+	for k, v := range config {
+		var entry = new(models.ConfigEntry)
+		entry.Key = k
+		switch v.(type) {
+		case string:
+			entry.Value=v.(string)
+		case int:
+			entry.Value=strconv.Itoa(v.(int))
+		case bool:
+			entry.Value=strconv.FormatBool(v.(bool))
+		case float64:
+			entry.Value=strconv.Itoa(int(v.(float64)))
+		default:
+			return nil, fmt.Errorf("unknown type %v", v)
+		}
+		configEntries = append(configEntries,*entry)
+	}
+	return configEntries,nil
+}

src/adminserver/systemcfg/store/database/driver_db_test.go

@@ -0,0 +1,74 @@
+package database
+
+import (
+	"testing"
+    "github.com/stretchr/testify/assert"
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common"
+)
+
+func TestCfgStore_Name(t *testing.T) {
+	driver,err := NewCfgStore()
+	if err != nil {
+		t.Fatalf("Failed to create db configuration store %v", err)
+	}
+	assert.Equal(t, name, driver.Name())
+}
+
+func TestWrapperConfig(t *testing.T) {
+	cfg:=[]*models.ConfigEntry{
+		{
+			Key:common.CfgExpiration,
+			Value:"500",
+		},
+		{
+			Key:common.WithNotary,
+			Value:"true",
+		},
+		{
+			Key:common.MySQLHost,
+			Value:"192.168.1.210",
+		},
+	}
+	result,err := WrapperConfig(cfg)
+	if err != nil {
+		t.Fatalf("Failed to wrapper config %v", err)
+	}
+	withNotary,_ := result[common.WithNotary].(bool)
+	assert.Equal(t,true, withNotary)
+
+	mysqlhost, ok := result[common.MySQLHost].(string)
+	assert.True(t, ok)
+	assert.Equal(t, "192.168.1.210", mysqlhost)
+
+	expiration, ok := result[common.CfgExpiration].(float64)
+
+	assert.True(t, ok)
+	assert.Equal(t, float64(500), expiration)
+}
+
+func TestTranslateConfig(t *testing.T) {
+	config := map[string]interface{}{}
+	config[common.MySQLHost]="192.168.1.210"
+
+	entries,err := TranslateConfig(config)
+	if err != nil {
+		t.Fatalf("Failed to translate configuration %v", err)
+	}
+	assert.Equal(t, "192.168.1.210",entries[0].Value)
+	config =make(map[string]interface{})
+	config[common.WithNotary]=true
+	entries,err = TranslateConfig(config)
+	if err != nil {
+		t.Fatalf("Failed to translate configuration %v", err)
+	}
+	assert.Equal(t, "true", entries[0].Value)
+
+	config =make(map[string]interface{})
+	config[common.CfgExpiration]=float64(500)
+	entries,err = TranslateConfig(config)
+	if err != nil {
+		t.Fatalf("Failed to translate configuration %v", err)
+	}
+	assert.Equal(t, "500", entries[0].Value)
+}

src/adminserver/systemcfg/systemcfg.go

@@ -23,10 +23,13 @@ import (
 	enpt "github.com/vmware/harbor/src/adminserver/systemcfg/encrypt"
 	"github.com/vmware/harbor/src/adminserver/systemcfg/store"
 	"github.com/vmware/harbor/src/adminserver/systemcfg/store/encrypt"
-	"github.com/vmware/harbor/src/adminserver/systemcfg/store/json"
 	"github.com/vmware/harbor/src/common"
 	comcfg "github.com/vmware/harbor/src/common/config"
 	"github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/adminserver/systemcfg/store/database"
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/dao"
+	"github.com/vmware/harbor/src/adminserver/systemcfg/store/json"
 )
 
 const (
@@ -215,17 +218,61 @@ func Init() (err error) {
 }
 
 func initCfgStore() (err error) {
+
+	drivertype := os.Getenv("CFG_DRIVER")
+	if len(drivertype) == 0 {
+		drivertype = common.CfgDriverDB
+	}
 	path := os.Getenv("JSON_CFG_STORE_PATH")
 	if len(path) == 0 {
 		path = defaultJSONCfgStorePath
 	}
 	log.Infof("the path of json configuration storage: %s", path)
 
-	CfgStore, err = json.NewCfgStore(path)
-	if err != nil {
-		return
+	if drivertype == common.CfgDriverDB {
+		//init database
+		cfgs := map[string]interface{}{}
+		if err = LoadFromEnv(cfgs, true); err != nil {
+			return err
+		}
+		cfgdb := GetDatabaseFromCfg(cfgs)
+		if err = dao.InitDatabase(cfgdb); err != nil {
+			return err
+		}
+		CfgStore, err = database.NewCfgStore()
+		if err != nil {
+			return err
+		}
+		//migration check: if no data in the db , then will try to load from path
+		m, err := CfgStore.Read()
+		if err != nil {
+			return err
+		}
+		if m == nil || len(m) == 0 {
+			if _, err := os.Stat(path); err == nil {
+				jsondriver, err := json.NewCfgStore(path)
+				if err != nil {
+					log.Errorf("Failed to migrate configuration from %s", path)
+					return err
+				}
+				jsonconfig, err := jsondriver.Read()
+				if err != nil {
+					log.Errorf("Failed to read old configuration from %s", path)
+					return err
+				}
+				err = CfgStore.Write(jsonconfig)
+				if err != nil {
+					log.Error("Failed to update old configuration to dattabase")
+					return err
+				}
+			}
+		}
+	} else {
+		CfgStore, err = json.NewCfgStore(path)
+		if err != nil {
+			return err
+		}
 	}
-
 	kp := os.Getenv("KEY_PATH")
 	if len(kp) == 0 {
 		kp = defaultKeyPath
@@ -278,3 +325,20 @@ func LoadFromEnv(cfgs map[string]interface{}, all bool) error {
 
 	return nil
 }
+
+// GetDatabaseFromCfg Create database object from config
+func GetDatabaseFromCfg(cfg map[string]interface{}) (*models.Database){
+	database := &models.Database{}
+	database.Type = cfg[common.DatabaseType].(string)
+	mysql := &models.MySQL{}
+	mysql.Host = cfg[common.MySQLHost].(string)
+	mysql.Port = int(cfg[common.MySQLPort].(int))
+	mysql.Username = cfg[common.MySQLUsername].(string)
+	mysql.Password = cfg[common.MySQLPassword].(string)
+	mysql.Database = cfg[common.MySQLDatabase].(string)
+	database.MySQL = mysql
+	sqlite := &models.SQLite{}
+	sqlite.File = cfg[common.SQLiteFile].(string)
+	database.SQLite = sqlite
+	return database
+}

src/adminserver/systemcfg/systemcfg_test.go

@@ -62,6 +62,9 @@ func TestParseStringToBool(t *testing.T) {
 func TestInitCfgStore(t *testing.T) {
 	os.Clearenv()
 	path := "/tmp/config.json"
+	if err := os.Setenv("CFG_DRIVER", "json"); err != nil {
+		t.Fatalf("failed to set env: %v", err)
+	}
 	if err := os.Setenv("JSON_CFG_STORE_PATH", path); err != nil {
 		t.Fatalf("failed to set env: %v", err)
 	}
@@ -122,3 +125,19 @@ func TestLoadFromEnv(t *testing.T) {
 	assert.Equal(t, "ldap_url", cfgs[common.LDAPURL])
 	assert.Equal(t, true, cfgs[common.LDAPVerifyCert])
 }
+
+func TestGetDatabaseFromCfg(t *testing.T) {
+	cfg  :=map[string]interface{} {
+		common.DatabaseType:"mysql",
+		common.MySQLDatabase:"registry",
+		common.MySQLHost:"127.0.0.1",
+		common.MySQLPort:3306,
+		common.MySQLPassword:"1234",
+		common.MySQLUsername:"root",
+		common.SQLiteFile:"/tmp/sqlite.db",
+	}
+
+	database := GetDatabaseFromCfg(cfg)
+
+	assert.Equal(t,"mysql",database.Type)
+}

src/common/api/base.go

@@ -121,7 +121,8 @@ func (b *BaseAPI) RenderError(code int, text string) {
 func (b *BaseAPI) DecodeJSONReq(v interface{}) {
 	err := json.Unmarshal(b.Ctx.Input.CopyBody(1<<32), v)
 	if err != nil {
-		log.Errorf("Error while decoding the json request, error: %v", err)
+		log.Errorf("Error while decoding the json request, error: %v, %v",
+			err, string(b.Ctx.Input.CopyBody(1<<32)[:]))
 		b.CustomAbort(http.StatusBadRequest, "Invalid json request")
 	}
 }

src/common/const.go

@@ -73,6 +73,7 @@ const (
 	UAAEndpoint                = "uaa_endpoint"
 	UAAClientID                = "uaa_client_id"
 	UAAClientSecret            = "uaa_client_secret"
-
 	DefaultClairEndpoint = "http://clair:6060"
+	CfgDriverDB                = "db"
+	CfgDriverJSON              = "json"
 )

src/common/dao/config.go

@@ -29,3 +29,42 @@ func AuthModeCanBeModified() (bool, error) {
 	// admin and anonymous
 	return c == 2, nil
 }
+
+// GetConfigEntries Get configuration from database
+func GetConfigEntries() ([]*models.ConfigEntry, error) {
+	o := GetOrmer()
+	var p []*models.ConfigEntry
+	sql:="select * from properties"
+	n,err := o.Raw(sql,[]interface{}{}).QueryRows(&p)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if n == 0 {
+		return nil, nil
+	}
+	return p,nil
+}
+
+// SaveConfigEntries Save configuration to database.
+func SaveConfigEntries(entries []models.ConfigEntry) error{
+	o := GetOrmer()
+	tempEntry:=models.ConfigEntry{}
+	for _, entry := range entries{
+		tempEntry.Key = entry.Key
+		tempEntry.Value = entry.Value
+		created, _, error := o.ReadOrCreate(&tempEntry,"k")
+		if error != nil {
+			return error
+		}
+		if !created {
+			entry.ID = tempEntry.ID
+			_ ,err := o.Update(&entry,"v")
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

src/common/dao/config_test.go

@@ -68,4 +68,4 @@ func TestAuthModeCanBeModified(t *testing.T) {
 			t.Errorf("unexpected result: %t != %t", flag, false)
 		}
 	}
-}
+}

src/common/dao/dao_test.go

@@ -20,7 +20,7 @@ import (
 	"time"
 
 	"github.com/astaxie/beego/orm"
-	//"github.com/vmware/harbor/src/common/config"
+	"github.com/vmware/harbor/src/common"
 	"github.com/stretchr/testify/assert"
 	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/utils"
@@ -1631,3 +1631,103 @@ func TestGetScanJobsByStatus(t *testing.T) {
 	assert.Equal(1, len(r2))
 	assert.Equal(sj1.Repository, r2[0].Repository)
 }
+
+
+func TestSaveConfigEntries(t *testing.T)  {
+	configEntries :=[]models.ConfigEntry{
+		{
+			Key:"teststringkey",
+			Value:"192.168.111.211",
+		},
+		{
+			Key:"testboolkey",
+			Value:"true",
+		},
+		{
+			Key:"testnumberkey",
+			Value:"5",
+		},
+		{
+			Key:common.CfgDriverDB,
+			Value:"db",
+		},
+	}
+	err := SaveConfigEntries(configEntries)
+	if err != nil {
+		t.Fatalf("failed to save configuration to database %v", err)
+	}
+	readEntries, err:=GetConfigEntries()
+	if err !=nil {
+		t.Fatalf("Failed to get configuration from database %v", err)
+	}
+	findItem:=0
+	for _,entry:= range readEntries{
+		switch entry.Key {
+		case "teststringkey":
+			if "192.168.111.211" == entry.Value {
+				findItem++
+			}
+		case "testnumberkey":
+			if "5" == entry.Value {
+				findItem++
+			}
+		case "testboolkey":
+			if "true" == entry.Value {
+				findItem++
+			}
+		default:
+		}
+	}
+	if findItem !=3 {
+		t.Fatalf("Should update 3 configuration but only update %d", findItem)
+	}
+
+	configEntries =[]models.ConfigEntry{
+		{
+			Key:"teststringkey",
+			Value:"192.168.111.215",
+		},
+		{
+			Key:"testboolkey",
+			Value:"false",
+		},
+		{
+			Key:"testnumberkey",
+			Value:"7",
+		},
+		{
+			Key:common.CfgDriverDB,
+			Value:"db",
+		},
+	}
+	err = SaveConfigEntries(configEntries)
+	if err != nil {
+		t.Fatalf("failed to save configuration to database %v", err)
+	}
+	readEntries, err=GetConfigEntries()
+	if err !=nil {
+		t.Fatalf("Failed to get configuration from database %v", err)
+	}
+	findItem=0
+	for _,entry:= range readEntries{
+		switch entry.Key {
+		case "teststringkey":
+			if "192.168.111.215" == entry.Value {
+				findItem++
+			}
+		case "testnumberkey":
+			if "7" == entry.Value {
+				findItem++
+			}
+		case "testboolkey":
+			if "false" == entry.Value {
+				findItem++
+			}
+		default:
+		}
+	}
+	if findItem !=3 {
+		t.Fatalf("Should update 3 configuration but only update %d", findItem)
+	}
+
+}

src/common/models/base.go

@@ -30,5 +30,6 @@ func init() {
 		new(RepoRecord),
 		new(ImgScanOverview),
 		new(ClairVulnTimestamp),
-		new(ProjectMetadata))
+		new(ProjectMetadata),
+	    new(ConfigEntry))
 }

src/common/models/config.go

@@ -98,3 +98,14 @@ type SystemCfg struct {
 	CfgExpiration              int             `json:"cfg_expiration"`
 }
 */
+
+// ConfigEntry ...
+type ConfigEntry struct {
+	ID   int64    `orm:"pk;auto;column(id)" json:"-"`
+	Key string `orm:"column(k)" json:"k"`
+	Value string `orm:"column(v)" json:"v"`
+}
+// TableName ...
+func (ce *ConfigEntry)TableName() string {
+	return "properties"
+}

tests/testprepare.sh

@@ -7,9 +7,10 @@ cp make/common/config/ui/private_key.pem /etc/ui/.
 
 mkdir conf
 cp make/common/config/ui/app.conf conf/.
-
-sed -i -r "s/MYSQL_HOST=mysql/MYSQL_HOST=127.0.0.1/" make/common/config/adminserver/env
-sed -i -r "s|REGISTRY_URL=http://registry:5000|REGISTRY_URL=http://127.0.0.1:5000|" make/common/config/adminserver/env
+IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'`
+echo "server ip is "$IP
+sed -i -r "s/MYSQL_HOST=mysql/MYSQL_HOST=$IP/" make/common/config/adminserver/env
+sed -i -r "s|REGISTRY_URL=http://registry:5000|REGISTRY_URL=http://$IP:5000|" make/common/config/adminserver/env
 sed -i -r "s/UI_SECRET=.*/UI_SECRET=$UI_SECRET/" make/common/config/adminserver/env
 
-chmod 777 /data/
+chmod 777 /data/

tools/migration/changelog.md

@@ -56,3 +56,6 @@ Changelog for harbor database schema
   - insert data into table `project_metadata`
   - delete column `public` from table `project`
   - add column `insecure` to table `replication_target`
+## 1.3.x
+  - add pk `id` to table `properties`
+  - remove pk index from colum 'k' of table `properties`