Merge pull request #12674 from reasonerjt/standalone-db-migrator

Provide a standalone migrator to migrate DB schema.
This commit is contained in:
Wenkai Yin(尹文开) 2020-08-10 15:11:52 +08:00 committed by GitHub
commit e8f9fb63c0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 189 additions and 22 deletions

View File

@ -177,11 +177,13 @@ GOBUILDPATH_CORE=$(GOBUILDPATHINCONTAINER)/src/core
GOBUILDPATH_JOBSERVICE=$(GOBUILDPATHINCONTAINER)/src/jobservice
GOBUILDPATH_REGISTRYCTL=$(GOBUILDPATHINCONTAINER)/src/registryctl
GOBUILDPATH_MIGRATEPATCH=$(GOBUILDPATHINCONTAINER)/src/cmd/migrate-patch
GOBUILDPATH_STANDALONE_DB_MIGRATOR=$(GOBUILDPATHINCONTAINER)/src/cmd/standalone-db-migrator
GOBUILDMAKEPATH=make
GOBUILDMAKEPATH_CORE=$(GOBUILDMAKEPATH)/photon/core
GOBUILDMAKEPATH_JOBSERVICE=$(GOBUILDMAKEPATH)/photon/jobservice
GOBUILDMAKEPATH_REGISTRYCTL=$(GOBUILDMAKEPATH)/photon/registryctl
GOBUILDMAKEPATH_NOTARY=$(GOBUILDMAKEPATH)/photon/notary
GOBUILDMAKEPATH_STANDALONE_DB_MIGRATOR=$(GOBUILDMAKEPATH)/photon/standalone-db-migrator
# binary
CORE_BINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_CORE)
@ -192,6 +194,8 @@ REGISTRYCTLBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_REGISTRYCTL)
REGISTRYCTLBINARYNAME=harbor_registryctl
MIGRATEPATCHBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_NOTARY)
MIGRATEPATCHBINARYNAME=migrate-patch
STANDALONE_DB_MIGRATOR_BINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_STANDALONE_DB_MIGRATOR)
STANDALONE_DB_MIGRATOR_BINARYNAME=migrate
# configfile
CONFIGPATH=$(MAKEPATH)
@ -378,6 +382,11 @@ compile_notary_migrate_patch:
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_MIGRATEPATCH) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_NOTARY)/$(MIGRATEPATCHBINARYNAME)
@echo "Done."
compile_standalone_db_migrator:
@echo "compiling binary for standalone db migrator (golang image)..."
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_STANDALONE_DB_MIGRATOR) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_STANDALONE_DB_MIGRATOR)/$(STANDALONE_DB_MIGRATOR_BINARYNAME)
@echo "Done."
compile: check_environment versions_prepare compile_core compile_jobservice compile_registryctl compile_notary_migrate_patch
update_prepare_version:
@ -406,6 +415,9 @@ build:
-e CLAIRURL=$(CLAIRURL) -e CHARTURL=$(CHARTURL) -e NORARYURL=$(NORARYURL) -e REGISTRYURL=$(REGISTRYURL) -e CLAIR_ADAPTER_DOWNLOAD_URL=$(CLAIR_ADAPTER_DOWNLOAD_URL) \
-e TRIVY_DOWNLOAD_URL=$(TRIVY_DOWNLOAD_URL) -e TRIVY_ADAPTER_DOWNLOAD_URL=$(TRIVY_ADAPTER_DOWNLOAD_URL)
build_standalone_db_migrator: compile_standalone_db_migrator
make -f $(MAKEFILEPATH_PHOTON)/Makefile _build_standalone_db_migrator -e BASEIMAGETAG=$(BASEIMAGETAG) -e VERSIONTAG=$(VERSIONTAG)
build_base_docker:
@for name in chartserver clair clair-adapter trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
echo $$name ; \

View File

@ -95,6 +95,11 @@ DOCKERFILEPATH_REDIS=$(DOCKERFILEPATH)/redis
DOCKERFILENAME_REDIS=Dockerfile
DOCKERIMAGENAME_REDIS=goharbor/redis-photon
DOCKERFILEPATH_STANDALONE_DB_MIGRATOR=$(DOCKERFILEPATH)/standalone-db-migrator
DOCKERFILENAME_STANDALONE_DB_MIGRATOR=Dockerfile
DOCKERIMAGENAME_STANDALONE_DB_MIGRATOR=goharbor/standalone-db-migrator
# for chart server (chartmuseum)
DOCKERFILEPATH_CHART_SERVER=$(DOCKERFILEPATH)/chartserver
DOCKERFILENAME_CHART_SERVER=Dockerfile
@ -243,6 +248,11 @@ _build_redis:
@$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_REDIS)/$(DOCKERFILENAME_REDIS) -t $(DOCKERIMAGENAME_REDIS):$(VERSIONTAG) .
@echo "Done."
_build_standalone_db_migrator:
@echo "building standalone db migrator image for photon..."
@$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_STANDALONE_DB_MIGRATOR)/$(DOCKERFILENAME_STANDALONE_DB_MIGRATOR) -t $(DOCKERIMAGENAME_STANDALONE_DB_MIGRATOR):$(VERSIONTAG) .
@echo "Done."
define _extract_archive
echo "download $1";\
$(CURL) --connect-timeout 30 -f -k -L $1 | tar xvz -C $2 || exit 1

View File

@ -0,0 +1,17 @@
ARG harbor_base_image_version
ARG harbor_base_namespace
FROM ${harbor_base_namespace}/harbor-db-base:${harbor_base_image_version}
ENV EXTERNAL_DB 0
RUN mkdir /harbor/
COPY ./make/migrations /migrations
COPY ./make/photon/standalone-db-migrator/migrate /harbor/
COPY ./make/photon/standalone-db-migrator/entrypoint.sh /harbor/
RUN chown -R postgres:postgres /harbor/ \
&& chown -R postgres:postgres /migrations/ \
&& chmod u+x /harbor/migrate /harbor/entrypoint.sh
USER postgres
ENTRYPOINT ["/harbor/entrypoint.sh"]

View File

@ -0,0 +1,6 @@
#!/bin/bash
set -e
[ $EXTERNAL_DB -eq 1 ] || pg_ctl start -w -t 60 -D ${PGDATA}
/harbor/migrate
[ $EXTERNAL_DB -eq 1 ] || pg_ctl stop -D ${PGDATA}

View File

@ -0,0 +1,58 @@
package main
import (
"os"
"strconv"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/migration"
)
// key: env var, value: default value
var defaultAttrs = map[string]string{
"POSTGRESQL_HOST": "localhost",
"POSTGRESQL_PORT": "5432",
"POSTGRESQL_USERNAME": "postgres",
"POSTGRESQL_PASSWORD": "password",
"POSTGRESQL_DATABASE": "registry",
"POSTGRESQL_SSLMODE": "disable",
}
func main() {
p, _ := strconv.Atoi(getAttr("POSTGRESQL_PORT"))
db := &models.Database{
Type: "postgresql",
PostGreSQL: &models.PostGreSQL{
Host: getAttr("POSTGRESQL_HOST"),
Port: p,
Username: getAttr("POSTGRESQL_USERNAME"),
Password: getAttr("POSTGRESQL_PASSWORD"),
Database: getAttr("POSTGRESQL_DATABASE"),
SSLMode: getAttr("POSTGRESQL_SSLMODE"),
MaxIdleConns: 5,
MaxOpenConns: 5,
},
}
log.Info("Migrating the data to latest schema...")
log.Infof("DB info: postgres://%s@%s:%d/%s?sslmode=%s", db.PostGreSQL.Username, db.PostGreSQL.Host,
db.PostGreSQL.Port, db.PostGreSQL.Database, db.PostGreSQL.SSLMode)
if err := dao.InitDatabase(db); err != nil {
log.Fatalf("failed to initialize database: %v", err)
}
if err := migration.MigrateDB(db); err != nil {
log.Fatalf("failed to migrate DB: %v", err)
}
log.Info("Migration done. The data schema in DB is now update to date.")
}
func getAttr(k string) string {
v := os.Getenv(k)
if len(v) > 0 {
return v
}
return defaultAttrs[k]
}

View File

@ -153,3 +153,24 @@ func Escape(str string) string {
str = strings.Replace(str, `_`, `\_`, -1)
return str
}
// implements github.com/golang-migrate/migrate/v4.Logger
type mLogger struct {
logger *log.Logger
}
func newMigrateLogger() *mLogger {
return &mLogger{
logger: log.DefaultLogger().WithDepth(5),
}
}
// Verbose ...
func (l *mLogger) Verbose() bool {
return l.logger.GetLevel() <= log.DebugLevel
}
// Printf ...
func (l *mLogger) Printf(format string, v ...interface{}) {
l.logger.Infof(format, v...)
}

View File

@ -0,0 +1,17 @@
package dao
import (
"testing"
"github.com/goharbor/harbor/src/lib/log"
"github.com/stretchr/testify/assert"
)
func TestMLogger_Verbose(t *testing.T) {
l := newMigrateLogger()
if log.DefaultLogger().GetLevel() <= log.DebugLevel {
assert.True(t, l.Verbose())
} else {
assert.False(t, l.Verbose())
}
}

View File

@ -151,5 +151,10 @@ func NewMigrator(database *models.PostGreSQL) (*migrate.Migrate, error) {
path = defaultMigrationPath
}
srcURL := fmt.Sprintf("file://%s", path)
return migrate.New(srcURL, dbURL.String())
m, err := migrate.New(srcURL, dbURL.String())
if err != nil {
return nil, err
}
m.Log = newMigrateLogger()
return m, nil
}

View File

@ -261,6 +261,11 @@ func (l *Logger) Fatalf(format string, v ...interface{}) {
os.Exit(1)
}
// GetLevel returns the verbosity level of this logger
func (l *Logger) GetLevel() Level {
return l.lvl
}
func (l *Logger) getLine() string {
var str string
if !l.skipLine {
@ -326,6 +331,11 @@ func Fatalf(format string, v ...interface{}) {
logger.WithDepth(4).Fatalf(format, v...)
}
// GetLevel return the verbosity level of default logger
func GetLevel() Level {
return logger.GetLevel()
}
func line(callDepth int) string {
_, file, line, ok := runtime.Caller(callDepth)
if !ok {

View File

@ -25,7 +25,7 @@ import (
"github.com/goharbor/harbor/src/pkg/repository"
)
func upgradeData(ctx context.Context) error {
func abstractArtData(ctx context.Context) error {
abstractor := art.NewAbstractor()
pros, err := project.Mgr.List(ctx)
if err != nil {
@ -52,13 +52,13 @@ func upgradeData(ctx context.Context) error {
log.Errorf("failed to list artifacts under the repository %s: %v, skip", repo.Name, err)
continue
}
for _, art := range arts {
if err = abstract(ctx, abstractor, art); err != nil {
log.Errorf("failed to abstract the artifact %s@%s: %v, skip", art.RepositoryName, art.Digest, err)
for _, a := range arts {
if err = abstract(ctx, abstractor, a); err != nil {
log.Errorf("failed to abstract the artifact %s@%s: %v, skip", a.RepositoryName, a.Digest, err)
continue
}
if err = artifact.Mgr.Update(ctx, art); err != nil {
log.Errorf("failed to update the artifact %s@%s: %v, skip", repo.Name, art.Digest, err)
if err = artifact.Mgr.Update(ctx, a); err != nil {
log.Errorf("failed to update the artifact %s@%s: %v, skip", repo.Name, a.Digest, err)
continue
}
}
@ -67,7 +67,7 @@ func upgradeData(ctx context.Context) error {
}
// update data version
return setDataVersion(ctx, 30)
return setDataVersion(ctx, dataversionV2_0_0)
}
func abstract(ctx context.Context, abstractor art.Abstractor, art *artifact.Artifact) error {

View File

@ -25,30 +25,36 @@ import (
"github.com/golang-migrate/migrate/v4"
)
// Migrate the database schema and data
func Migrate(database *models.Database) error {
const (
schemaVersionV1_10_0 = 15
// data version for tracking the data integrity in the DB, it can be different from schema version
dataversionV2_0_0 = 30
)
// MigrateDB upgrades DB schema and do necessary transformation of the data in DB
func MigrateDB(database *models.Database) error {
// check the database schema version
migrator, err := dao.NewMigrator(database.PostGreSQL)
if err != nil {
return err
}
defer migrator.Close()
schemaVersion, _, err := migrator.Version()
if err != nil && err != migrate.ErrNilVersion {
return err
}
log.Debugf("current database schema version: %v", schemaVersion)
// prior to 1.9, version = 0 means fresh install
if schemaVersion > 0 && schemaVersion < 10 {
return fmt.Errorf("please upgrade to version 1.9 first")
if schemaVersion > 0 && schemaVersion < schemaVersionV1_10_0 {
return fmt.Errorf("please upgrade to version 1.10 first")
}
// update database schema
if err := dao.UpgradeSchema(database); err != nil {
return err
}
return dao.UpgradeSchema(database)
}
// AbstractArtifactData accesses the registry to
func AbstractArtifactData() error {
log.Info("Abstracting artifact data to DB...")
ctx := orm.NewContext(context.Background(), beegorm.NewOrm())
dataVersion, err := getDataVersion(ctx)
if err != nil {
@ -56,16 +62,21 @@ func Migrate(database *models.Database) error {
}
log.Debugf("current data version: %v", dataVersion)
// the abstract logic already done before, skip
if dataVersion == 30 {
log.Debug("no change in data, skip")
if dataVersion >= dataversionV2_0_0 {
log.Info("No need to abstract artifact data. Skip")
return nil
}
return abstractArtData(ctx)
}
// upgrade data
if err = upgradeData(ctx); err != nil {
// Migrate the database schema and abstract artifact data
func Migrate(database *models.Database) error {
if err := MigrateDB(database); err != nil {
return err
}
if err := AbstractArtifactData(); err != nil {
return err
}
return nil
}