Merge pull request #12433 from heww/fix-db-max-open-conns

fix(db): set max open conns of sql.DB manually

src/common/dao/pgsql.go

@@ -22,13 +22,12 @@ import (
 
 	"github.com/astaxie/beego/orm"
 	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/common/utils"
+	"github.com/goharbor/harbor/src/lib/log"
 	migrate "github.com/golang-migrate/migrate/v4"
 	_ "github.com/golang-migrate/migrate/v4/database/postgres" // import pgsql driver for migrator
 	_ "github.com/golang-migrate/migrate/v4/source/file"       // import local file driver for migrator
-
-	"github.com/goharbor/harbor/src/common/utils"
-	"github.com/goharbor/harbor/src/lib/log"
-	_ "github.com/lib/pq" // register pgsql driver
+	_ "github.com/lib/pq"                                      // register pgsql driver
 )
 
 const defaultMigrationPath = "migrations/postgresql/"
@@ -89,7 +88,17 @@ func (p *pgsql) Register(alias ...string) error {
 	info := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
 		p.host, p.port, p.usr, p.pwd, p.database, p.sslmode)
 
-	return orm.RegisterDataBase(an, "postgres", info, p.maxIdleConns, p.maxOpenConns)
+	if err := orm.RegisterDataBase(an, "postgres", info, p.maxIdleConns, p.maxOpenConns); err != nil {
+		return err
+	}
+
+	// Due to the issues of beego v1.12.1 and v1.12.2, we set the max open conns ourselves.
+	// See https://github.com/goharbor/harbor/issues/12403
+	// and https://github.com/astaxie/beego/issues/4059 for more info.
+	db, _ := orm.GetDB(an)
+	db.SetMaxOpenConns(p.maxOpenConns)
+
+	return nil
 }
 
 // UpgradeSchema calls migrate tool to upgrade schema to the latest based on the SQL scripts.

src/common/dao/pgsql_test.go

@@ -0,0 +1,52 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/astaxie/beego/orm"
+)
+
+func TestMaxOpenConns(t *testing.T) {
+	var wg sync.WaitGroup
+
+	queryNum := 200
+	results := make([]bool, queryNum)
+	for i := 0; i < queryNum; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+
+			o := orm.NewOrm()
+			if _, err := o.Raw("SELECT pg_sleep(10)").Exec(); err != nil {
+				fmt.Printf("failed to get the count of the projects, error: %v\n", err)
+				results[i] = false
+			} else {
+				results[i] = true
+			}
+		}(i)
+	}
+
+	wg.Wait()
+
+	for _, success := range results {
+		if !success {
+			t.Fatal("max open conns not work")
+		}
+	}
+}

src/common/dao/testutils.go

@@ -61,11 +61,13 @@ func PrepareTestForPostgresSQL() {
 	database := &models.Database{
 		Type: "postgresql",
 		PostGreSQL: &models.PostGreSQL{
-			Host:     dbHost,
-			Port:     dbPort,
-			Username: dbUser,
-			Password: dbPassword,
-			Database: dbDatabase,
+			Host:         dbHost,
+			Port:         dbPort,
+			Username:     dbUser,
+			Password:     dbPassword,
+			Database:     dbDatabase,
+			MaxIdleConns: 50,
+			MaxOpenConns: 100,
 		},
 	}