Merge pull request #12674 from reasonerjt/standalone-db-migrator

Provide a standalone migrator to migrate DB schema.

Makefile

@@ -177,11 +177,13 @@ GOBUILDPATH_CORE=$(GOBUILDPATHINCONTAINER)/src/core
 GOBUILDPATH_JOBSERVICE=$(GOBUILDPATHINCONTAINER)/src/jobservice
 GOBUILDPATH_REGISTRYCTL=$(GOBUILDPATHINCONTAINER)/src/registryctl
 GOBUILDPATH_MIGRATEPATCH=$(GOBUILDPATHINCONTAINER)/src/cmd/migrate-patch
+GOBUILDPATH_STANDALONE_DB_MIGRATOR=$(GOBUILDPATHINCONTAINER)/src/cmd/standalone-db-migrator
 GOBUILDMAKEPATH=make
 GOBUILDMAKEPATH_CORE=$(GOBUILDMAKEPATH)/photon/core
 GOBUILDMAKEPATH_JOBSERVICE=$(GOBUILDMAKEPATH)/photon/jobservice
 GOBUILDMAKEPATH_REGISTRYCTL=$(GOBUILDMAKEPATH)/photon/registryctl
 GOBUILDMAKEPATH_NOTARY=$(GOBUILDMAKEPATH)/photon/notary
+GOBUILDMAKEPATH_STANDALONE_DB_MIGRATOR=$(GOBUILDMAKEPATH)/photon/standalone-db-migrator
 
 # binary
 CORE_BINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_CORE)
@@ -192,6 +194,8 @@ REGISTRYCTLBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_REGISTRYCTL)
 REGISTRYCTLBINARYNAME=harbor_registryctl
 MIGRATEPATCHBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_NOTARY)
 MIGRATEPATCHBINARYNAME=migrate-patch
+STANDALONE_DB_MIGRATOR_BINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_STANDALONE_DB_MIGRATOR)
+STANDALONE_DB_MIGRATOR_BINARYNAME=migrate
 
 # configfile
 CONFIGPATH=$(MAKEPATH)
@@ -378,6 +382,11 @@ compile_notary_migrate_patch:
 	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_MIGRATEPATCH) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_NOTARY)/$(MIGRATEPATCHBINARYNAME)
 	@echo "Done."
 
+compile_standalone_db_migrator:
+	@echo "compiling binary for standalone db migrator (golang image)..."
+	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_STANDALONE_DB_MIGRATOR) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_STANDALONE_DB_MIGRATOR)/$(STANDALONE_DB_MIGRATOR_BINARYNAME)
+	@echo "Done."
+
 compile: check_environment versions_prepare compile_core compile_jobservice compile_registryctl compile_notary_migrate_patch
 
 update_prepare_version:
@@ -406,6 +415,9 @@ build:
 	 -e CLAIRURL=$(CLAIRURL) -e CHARTURL=$(CHARTURL) -e NORARYURL=$(NORARYURL) -e REGISTRYURL=$(REGISTRYURL) -e CLAIR_ADAPTER_DOWNLOAD_URL=$(CLAIR_ADAPTER_DOWNLOAD_URL) \
 	 -e TRIVY_DOWNLOAD_URL=$(TRIVY_DOWNLOAD_URL) -e TRIVY_ADAPTER_DOWNLOAD_URL=$(TRIVY_ADAPTER_DOWNLOAD_URL)
 
+build_standalone_db_migrator: compile_standalone_db_migrator
+	make -f $(MAKEFILEPATH_PHOTON)/Makefile _build_standalone_db_migrator -e BASEIMAGETAG=$(BASEIMAGETAG) -e VERSIONTAG=$(VERSIONTAG)
+
 build_base_docker:
 	@for name in chartserver clair clair-adapter trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
 		echo $$name ; \

make/photon/Makefile

@@ -95,6 +95,11 @@ DOCKERFILEPATH_REDIS=$(DOCKERFILEPATH)/redis
 DOCKERFILENAME_REDIS=Dockerfile
 DOCKERIMAGENAME_REDIS=goharbor/redis-photon
 
+DOCKERFILEPATH_STANDALONE_DB_MIGRATOR=$(DOCKERFILEPATH)/standalone-db-migrator
+DOCKERFILENAME_STANDALONE_DB_MIGRATOR=Dockerfile
+DOCKERIMAGENAME_STANDALONE_DB_MIGRATOR=goharbor/standalone-db-migrator
+
+
 # for chart server (chartmuseum)
 DOCKERFILEPATH_CHART_SERVER=$(DOCKERFILEPATH)/chartserver
 DOCKERFILENAME_CHART_SERVER=Dockerfile
@@ -243,6 +248,11 @@ _build_redis:
 	@$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_REDIS)/$(DOCKERFILENAME_REDIS) -t $(DOCKERIMAGENAME_REDIS):$(VERSIONTAG) .
 	@echo "Done."
 
+_build_standalone_db_migrator:
+	@echo "building standalone db migrator image for photon..."
+	@$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_STANDALONE_DB_MIGRATOR)/$(DOCKERFILENAME_STANDALONE_DB_MIGRATOR) -t $(DOCKERIMAGENAME_STANDALONE_DB_MIGRATOR):$(VERSIONTAG) .
+	@echo "Done."
+
 define _extract_archive
 	echo "download $1";\
 	$(CURL) --connect-timeout 30 -f -k -L $1 | tar xvz -C $2 || exit 1

make/photon/standalone-db-migrator/Dockerfile

@@ -0,0 +1,17 @@
+ARG harbor_base_image_version
+ARG harbor_base_namespace
+FROM ${harbor_base_namespace}/harbor-db-base:${harbor_base_image_version}
+
+ENV EXTERNAL_DB 0
+
+RUN mkdir /harbor/
+COPY ./make/migrations /migrations
+COPY ./make/photon/standalone-db-migrator/migrate /harbor/
+COPY ./make/photon/standalone-db-migrator/entrypoint.sh /harbor/
+
+RUN chown -R postgres:postgres /harbor/ \
+    && chown -R postgres:postgres /migrations/ \
+    && chmod u+x /harbor/migrate /harbor/entrypoint.sh
+USER postgres
+
+ENTRYPOINT ["/harbor/entrypoint.sh"]

make/photon/standalone-db-migrator/entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+[ $EXTERNAL_DB -eq 1 ] || pg_ctl start -w -t 60 -D ${PGDATA}
+/harbor/migrate
+[ $EXTERNAL_DB -eq 1 ] || pg_ctl stop -D ${PGDATA}

src/cmd/standalone-db-migrator/main.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"os"
+	"strconv"
+
+	"github.com/goharbor/harbor/src/common/dao"
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/migration"
+)
+
+// key: env var, value: default value
+var defaultAttrs = map[string]string{
+	"POSTGRESQL_HOST":     "localhost",
+	"POSTGRESQL_PORT":     "5432",
+	"POSTGRESQL_USERNAME": "postgres",
+	"POSTGRESQL_PASSWORD": "password",
+	"POSTGRESQL_DATABASE": "registry",
+	"POSTGRESQL_SSLMODE":  "disable",
+}
+
+func main() {
+	p, _ := strconv.Atoi(getAttr("POSTGRESQL_PORT"))
+	db := &models.Database{
+		Type: "postgresql",
+		PostGreSQL: &models.PostGreSQL{
+			Host:         getAttr("POSTGRESQL_HOST"),
+			Port:         p,
+			Username:     getAttr("POSTGRESQL_USERNAME"),
+			Password:     getAttr("POSTGRESQL_PASSWORD"),
+			Database:     getAttr("POSTGRESQL_DATABASE"),
+			SSLMode:      getAttr("POSTGRESQL_SSLMODE"),
+			MaxIdleConns: 5,
+			MaxOpenConns: 5,
+		},
+	}
+
+	log.Info("Migrating the data to latest schema...")
+	log.Infof("DB info: postgres://%s@%s:%d/%s?sslmode=%s", db.PostGreSQL.Username, db.PostGreSQL.Host,
+		db.PostGreSQL.Port, db.PostGreSQL.Database, db.PostGreSQL.SSLMode)
+
+	if err := dao.InitDatabase(db); err != nil {
+		log.Fatalf("failed to initialize database: %v", err)
+	}
+	if err := migration.MigrateDB(db); err != nil {
+		log.Fatalf("failed to migrate DB: %v", err)
+	}
+	log.Info("Migration done.  The data schema in DB is now update to date.")
+}
+
+func getAttr(k string) string {
+	v := os.Getenv(k)
+	if len(v) > 0 {
+		return v
+	}
+	return defaultAttrs[k]
+}

src/common/dao/base.go

@@ -153,3 +153,24 @@ func Escape(str string) string {
 	str = strings.Replace(str, `_`, `\_`, -1)
 	return str
 }
+
+// implements github.com/golang-migrate/migrate/v4.Logger
+type mLogger struct {
+	logger *log.Logger
+}
+
+func newMigrateLogger() *mLogger {
+	return &mLogger{
+		logger: log.DefaultLogger().WithDepth(5),
+	}
+}
+
+// Verbose ...
+func (l *mLogger) Verbose() bool {
+	return l.logger.GetLevel() <= log.DebugLevel
+}
+
+// Printf ...
+func (l *mLogger) Printf(format string, v ...interface{}) {
+	l.logger.Infof(format, v...)
+}

src/common/dao/base_test.go

@@ -0,0 +1,17 @@
+package dao
+
+import (
+	"testing"
+
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMLogger_Verbose(t *testing.T) {
+	l := newMigrateLogger()
+	if log.DefaultLogger().GetLevel() <= log.DebugLevel {
+		assert.True(t, l.Verbose())
+	} else {
+		assert.False(t, l.Verbose())
+	}
+}

src/common/dao/pgsql.go

@@ -151,5 +151,10 @@ func NewMigrator(database *models.PostGreSQL) (*migrate.Migrate, error) {
 		path = defaultMigrationPath
 	}
 	srcURL := fmt.Sprintf("file://%s", path)
-	return migrate.New(srcURL, dbURL.String())
+	m, err := migrate.New(srcURL, dbURL.String())
+	if err != nil {
+		return nil, err
+	}
+	m.Log = newMigrateLogger()
+	return m, nil
 }

src/lib/log/logger.go

@@ -261,6 +261,11 @@ func (l *Logger) Fatalf(format string, v ...interface{}) {
 	os.Exit(1)
 }
 
+// GetLevel returns the verbosity level of this logger
+func (l *Logger) GetLevel() Level {
+	return l.lvl
+}
+
 func (l *Logger) getLine() string {
 	var str string
 	if !l.skipLine {
@@ -326,6 +331,11 @@ func Fatalf(format string, v ...interface{}) {
 	logger.WithDepth(4).Fatalf(format, v...)
 }
 
+// GetLevel return the verbosity level of default logger
+func GetLevel() Level {
+	return logger.GetLevel()
+}
+
 func line(callDepth int) string {
 	_, file, line, ok := runtime.Caller(callDepth)
 	if !ok {

src/migration/artifact.go

@@ -25,7 +25,7 @@ import (
 	"github.com/goharbor/harbor/src/pkg/repository"
 )
 
-func upgradeData(ctx context.Context) error {
+func abstractArtData(ctx context.Context) error {
 	abstractor := art.NewAbstractor()
 	pros, err := project.Mgr.List(ctx)
 	if err != nil {
@@ -52,13 +52,13 @@ func upgradeData(ctx context.Context) error {
 				log.Errorf("failed to list artifacts under the repository %s: %v, skip", repo.Name, err)
 				continue
 			}
-			for _, art := range arts {
+			for _, a := range arts {
-				if err = abstract(ctx, abstractor, art); err != nil {
+				if err = abstract(ctx, abstractor, a); err != nil {
-					log.Errorf("failed to abstract the artifact %s@%s: %v, skip", art.RepositoryName, art.Digest, err)
+					log.Errorf("failed to abstract the artifact %s@%s: %v, skip", a.RepositoryName, a.Digest, err)
 					continue
 				}
-				if err = artifact.Mgr.Update(ctx, art); err != nil {
+				if err = artifact.Mgr.Update(ctx, a); err != nil {
-					log.Errorf("failed to update the artifact %s@%s: %v, skip", repo.Name, art.Digest, err)
+					log.Errorf("failed to update the artifact %s@%s: %v, skip", repo.Name, a.Digest, err)
 					continue
 				}
 			}
@@ -67,7 +67,7 @@ func upgradeData(ctx context.Context) error {
 	}
 
 	// update data version
-	return setDataVersion(ctx, 30)
+	return setDataVersion(ctx, dataversionV2_0_0)
 }
 
 func abstract(ctx context.Context, abstractor art.Abstractor, art *artifact.Artifact) error {

src/migration/migration.go

@@ -25,30 +25,36 @@ import (
 	"github.com/golang-migrate/migrate/v4"
 )
 
-// Migrate the database schema and data
+const (
-func Migrate(database *models.Database) error {
+	schemaVersionV1_10_0 = 15
+	// data version for tracking the data integrity in the DB, it can be different from schema version
+	dataversionV2_0_0 = 30
+)
+
+// MigrateDB upgrades DB schema and do necessary transformation of the data in DB
+func MigrateDB(database *models.Database) error {
 	// check the database schema version
 	migrator, err := dao.NewMigrator(database.PostGreSQL)
 	if err != nil {
 		return err
 	}
 	defer migrator.Close()
-
 	schemaVersion, _, err := migrator.Version()
 	if err != nil && err != migrate.ErrNilVersion {
 		return err
 	}
 	log.Debugf("current database schema version: %v", schemaVersion)
 	// prior to 1.9, version = 0 means fresh install
-	if schemaVersion > 0 && schemaVersion < 10 {
+	if schemaVersion > 0 && schemaVersion < schemaVersionV1_10_0 {
-		return fmt.Errorf("please upgrade to version 1.9 first")
+		return fmt.Errorf("please upgrade to version 1.10 first")
 	}
-
 	// update database schema
-	if err := dao.UpgradeSchema(database); err != nil {
+	return dao.UpgradeSchema(database)
-		return err
+}
-	}
 
+// AbstractArtifactData accesses the registry to
+func AbstractArtifactData() error {
+	log.Info("Abstracting artifact data to DB...")
 	ctx := orm.NewContext(context.Background(), beegorm.NewOrm())
 	dataVersion, err := getDataVersion(ctx)
 	if err != nil {
@@ -56,16 +62,21 @@ func Migrate(database *models.Database) error {
 	}
 	log.Debugf("current data version: %v", dataVersion)
 	// the abstract logic already done before, skip
-	if dataVersion == 30 {
+	if dataVersion >= dataversionV2_0_0 {
-		log.Debug("no change in data, skip")
+		log.Info("No need to abstract artifact data. Skip")
 		return nil
 	}
+	return abstractArtData(ctx)
+}
 
-	// upgrade data
+// Migrate the database schema and abstract artifact data
-	if err = upgradeData(ctx); err != nil {
+func Migrate(database *models.Database) error {
+	if err := MigrateDB(database); err != nil {
+		return err
+	}
+	if err := AbstractArtifactData(); err != nil {
 		return err
 	}
-
 	return nil
 }