From 4f94f59d2a67e9d6d4b0a0c7216e5aadcf1ded16 Mon Sep 17 00:00:00 2001 From: Daniel Jiang Date: Mon, 13 Jul 2020 16:57:09 +0800 Subject: [PATCH] Provide a standalone migrator to migrate DB schema. Fixes #11885 This part will not by default be packaged into release. A README.md will be added in another commit. Signed-off-by: Daniel Jiang --- Makefile | 12 ++++ make/photon/Makefile | 10 ++++ make/photon/standalone-db-migrator/Dockerfile | 17 ++++++ .../standalone-db-migrator/entrypoint.sh | 6 ++ src/cmd/standalone-db-migrator/main.go | 58 +++++++++++++++++++ src/common/dao/base.go | 21 +++++++ src/common/dao/base_test.go | 17 ++++++ src/common/dao/pgsql.go | 7 ++- src/lib/log/logger.go | 10 ++++ src/migration/artifact.go | 14 ++--- src/migration/migration.go | 39 ++++++++----- 11 files changed, 189 insertions(+), 22 deletions(-) create mode 100644 make/photon/standalone-db-migrator/Dockerfile create mode 100644 make/photon/standalone-db-migrator/entrypoint.sh create mode 100644 src/cmd/standalone-db-migrator/main.go create mode 100644 src/common/dao/base_test.go diff --git a/Makefile b/Makefile index 89a78053c..eef77e746 100644 --- a/Makefile +++ b/Makefile @@ -177,11 +177,13 @@ GOBUILDPATH_CORE=$(GOBUILDPATHINCONTAINER)/src/core GOBUILDPATH_JOBSERVICE=$(GOBUILDPATHINCONTAINER)/src/jobservice GOBUILDPATH_REGISTRYCTL=$(GOBUILDPATHINCONTAINER)/src/registryctl GOBUILDPATH_MIGRATEPATCH=$(GOBUILDPATHINCONTAINER)/src/cmd/migrate-patch +GOBUILDPATH_STANDALONE_DB_MIGRATOR=$(GOBUILDPATHINCONTAINER)/src/cmd/standalone-db-migrator GOBUILDMAKEPATH=make GOBUILDMAKEPATH_CORE=$(GOBUILDMAKEPATH)/photon/core GOBUILDMAKEPATH_JOBSERVICE=$(GOBUILDMAKEPATH)/photon/jobservice GOBUILDMAKEPATH_REGISTRYCTL=$(GOBUILDMAKEPATH)/photon/registryctl GOBUILDMAKEPATH_NOTARY=$(GOBUILDMAKEPATH)/photon/notary +GOBUILDMAKEPATH_STANDALONE_DB_MIGRATOR=$(GOBUILDMAKEPATH)/photon/standalone-db-migrator # binary CORE_BINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_CORE) @@ -192,6 +194,8 @@ REGISTRYCTLBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_REGISTRYCTL) REGISTRYCTLBINARYNAME=harbor_registryctl MIGRATEPATCHBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_NOTARY) MIGRATEPATCHBINARYNAME=migrate-patch +STANDALONE_DB_MIGRATOR_BINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_STANDALONE_DB_MIGRATOR) +STANDALONE_DB_MIGRATOR_BINARYNAME=migrate # configfile CONFIGPATH=$(MAKEPATH) @@ -378,6 +382,11 @@ compile_notary_migrate_patch: @$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_MIGRATEPATCH) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_NOTARY)/$(MIGRATEPATCHBINARYNAME) @echo "Done." +compile_standalone_db_migrator: + @echo "compiling binary for standalone db migrator (golang image)..." + @$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_STANDALONE_DB_MIGRATOR) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_STANDALONE_DB_MIGRATOR)/$(STANDALONE_DB_MIGRATOR_BINARYNAME) + @echo "Done." + compile: check_environment versions_prepare compile_core compile_jobservice compile_registryctl compile_notary_migrate_patch update_prepare_version: @@ -406,6 +415,9 @@ build: -e CLAIRURL=$(CLAIRURL) -e CHARTURL=$(CHARTURL) -e NORARYURL=$(NORARYURL) -e REGISTRYURL=$(REGISTRYURL) -e CLAIR_ADAPTER_DOWNLOAD_URL=$(CLAIR_ADAPTER_DOWNLOAD_URL) \ -e TRIVY_DOWNLOAD_URL=$(TRIVY_DOWNLOAD_URL) -e TRIVY_ADAPTER_DOWNLOAD_URL=$(TRIVY_ADAPTER_DOWNLOAD_URL) +build_standalone_db_migrator: compile_standalone_db_migrator + make -f $(MAKEFILEPATH_PHOTON)/Makefile _build_standalone_db_migrator -e BASEIMAGETAG=$(BASEIMAGETAG) -e VERSIONTAG=$(VERSIONTAG) + build_base_docker: @for name in chartserver clair clair-adapter trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \ echo $$name ; \ diff --git a/make/photon/Makefile b/make/photon/Makefile index 8019e5f4d..44bb4818e 100644 --- a/make/photon/Makefile +++ b/make/photon/Makefile @@ -95,6 +95,11 @@ DOCKERFILEPATH_REDIS=$(DOCKERFILEPATH)/redis DOCKERFILENAME_REDIS=Dockerfile DOCKERIMAGENAME_REDIS=goharbor/redis-photon +DOCKERFILEPATH_STANDALONE_DB_MIGRATOR=$(DOCKERFILEPATH)/standalone-db-migrator +DOCKERFILENAME_STANDALONE_DB_MIGRATOR=Dockerfile +DOCKERIMAGENAME_STANDALONE_DB_MIGRATOR=goharbor/standalone-db-migrator + + # for chart server (chartmuseum) DOCKERFILEPATH_CHART_SERVER=$(DOCKERFILEPATH)/chartserver DOCKERFILENAME_CHART_SERVER=Dockerfile @@ -243,6 +248,11 @@ _build_redis: @$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_REDIS)/$(DOCKERFILENAME_REDIS) -t $(DOCKERIMAGENAME_REDIS):$(VERSIONTAG) . @echo "Done." +_build_standalone_db_migrator: + @echo "building standalone db migrator image for photon..." + @$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_STANDALONE_DB_MIGRATOR)/$(DOCKERFILENAME_STANDALONE_DB_MIGRATOR) -t $(DOCKERIMAGENAME_STANDALONE_DB_MIGRATOR):$(VERSIONTAG) . + @echo "Done." + define _extract_archive echo "download $1";\ $(CURL) --connect-timeout 30 -f -k -L $1 | tar xvz -C $2 || exit 1 diff --git a/make/photon/standalone-db-migrator/Dockerfile b/make/photon/standalone-db-migrator/Dockerfile new file mode 100644 index 000000000..0fc35c8a6 --- /dev/null +++ b/make/photon/standalone-db-migrator/Dockerfile @@ -0,0 +1,17 @@ +ARG harbor_base_image_version +ARG harbor_base_namespace +FROM ${harbor_base_namespace}/harbor-db-base:${harbor_base_image_version} + +ENV EXTERNAL_DB 0 + +RUN mkdir /harbor/ +COPY ./make/migrations /migrations +COPY ./make/photon/standalone-db-migrator/migrate /harbor/ +COPY ./make/photon/standalone-db-migrator/entrypoint.sh /harbor/ + +RUN chown -R postgres:postgres /harbor/ \ + && chown -R postgres:postgres /migrations/ \ + && chmod u+x /harbor/migrate /harbor/entrypoint.sh +USER postgres + +ENTRYPOINT ["/harbor/entrypoint.sh"] diff --git a/make/photon/standalone-db-migrator/entrypoint.sh b/make/photon/standalone-db-migrator/entrypoint.sh new file mode 100644 index 000000000..ad1fca068 --- /dev/null +++ b/make/photon/standalone-db-migrator/entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +[ $EXTERNAL_DB -eq 1 ] || pg_ctl start -w -t 60 -D ${PGDATA} +/harbor/migrate +[ $EXTERNAL_DB -eq 1 ] || pg_ctl stop -D ${PGDATA} diff --git a/src/cmd/standalone-db-migrator/main.go b/src/cmd/standalone-db-migrator/main.go new file mode 100644 index 000000000..db429423d --- /dev/null +++ b/src/cmd/standalone-db-migrator/main.go @@ -0,0 +1,58 @@ +package main + +import ( + "os" + "strconv" + + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/migration" +) + +// key: env var, value: default value +var defaultAttrs = map[string]string{ + "POSTGRESQL_HOST": "localhost", + "POSTGRESQL_PORT": "5432", + "POSTGRESQL_USERNAME": "postgres", + "POSTGRESQL_PASSWORD": "password", + "POSTGRESQL_DATABASE": "registry", + "POSTGRESQL_SSLMODE": "disable", +} + +func main() { + p, _ := strconv.Atoi(getAttr("POSTGRESQL_PORT")) + db := &models.Database{ + Type: "postgresql", + PostGreSQL: &models.PostGreSQL{ + Host: getAttr("POSTGRESQL_HOST"), + Port: p, + Username: getAttr("POSTGRESQL_USERNAME"), + Password: getAttr("POSTGRESQL_PASSWORD"), + Database: getAttr("POSTGRESQL_DATABASE"), + SSLMode: getAttr("POSTGRESQL_SSLMODE"), + MaxIdleConns: 5, + MaxOpenConns: 5, + }, + } + + log.Info("Migrating the data to latest schema...") + log.Infof("DB info: postgres://%s@%s:%d/%s?sslmode=%s", db.PostGreSQL.Username, db.PostGreSQL.Host, + db.PostGreSQL.Port, db.PostGreSQL.Database, db.PostGreSQL.SSLMode) + + if err := dao.InitDatabase(db); err != nil { + log.Fatalf("failed to initialize database: %v", err) + } + if err := migration.MigrateDB(db); err != nil { + log.Fatalf("failed to migrate DB: %v", err) + } + log.Info("Migration done. The data schema in DB is now update to date.") +} + +func getAttr(k string) string { + v := os.Getenv(k) + if len(v) > 0 { + return v + } + return defaultAttrs[k] +} diff --git a/src/common/dao/base.go b/src/common/dao/base.go index cbb3b873f..eb08d7d56 100644 --- a/src/common/dao/base.go +++ b/src/common/dao/base.go @@ -153,3 +153,24 @@ func Escape(str string) string { str = strings.Replace(str, `_`, `\_`, -1) return str } + +// implements github.com/golang-migrate/migrate/v4.Logger +type mLogger struct { + logger *log.Logger +} + +func newMigrateLogger() *mLogger { + return &mLogger{ + logger: log.DefaultLogger().WithDepth(5), + } +} + +// Verbose ... +func (l *mLogger) Verbose() bool { + return l.logger.GetLevel() <= log.DebugLevel +} + +// Printf ... +func (l *mLogger) Printf(format string, v ...interface{}) { + l.logger.Infof(format, v...) +} diff --git a/src/common/dao/base_test.go b/src/common/dao/base_test.go new file mode 100644 index 000000000..aa9bc7e60 --- /dev/null +++ b/src/common/dao/base_test.go @@ -0,0 +1,17 @@ +package dao + +import ( + "testing" + + "github.com/goharbor/harbor/src/lib/log" + "github.com/stretchr/testify/assert" +) + +func TestMLogger_Verbose(t *testing.T) { + l := newMigrateLogger() + if log.DefaultLogger().GetLevel() <= log.DebugLevel { + assert.True(t, l.Verbose()) + } else { + assert.False(t, l.Verbose()) + } +} diff --git a/src/common/dao/pgsql.go b/src/common/dao/pgsql.go index b6bc4f2d4..abf75aebe 100644 --- a/src/common/dao/pgsql.go +++ b/src/common/dao/pgsql.go @@ -151,5 +151,10 @@ func NewMigrator(database *models.PostGreSQL) (*migrate.Migrate, error) { path = defaultMigrationPath } srcURL := fmt.Sprintf("file://%s", path) - return migrate.New(srcURL, dbURL.String()) + m, err := migrate.New(srcURL, dbURL.String()) + if err != nil { + return nil, err + } + m.Log = newMigrateLogger() + return m, nil } diff --git a/src/lib/log/logger.go b/src/lib/log/logger.go index bb0b9c00f..438668131 100644 --- a/src/lib/log/logger.go +++ b/src/lib/log/logger.go @@ -261,6 +261,11 @@ func (l *Logger) Fatalf(format string, v ...interface{}) { os.Exit(1) } +// GetLevel returns the verbosity level of this logger +func (l *Logger) GetLevel() Level { + return l.lvl +} + func (l *Logger) getLine() string { var str string if !l.skipLine { @@ -326,6 +331,11 @@ func Fatalf(format string, v ...interface{}) { logger.WithDepth(4).Fatalf(format, v...) } +// GetLevel return the verbosity level of default logger +func GetLevel() Level { + return logger.GetLevel() +} + func line(callDepth int) string { _, file, line, ok := runtime.Caller(callDepth) if !ok { diff --git a/src/migration/artifact.go b/src/migration/artifact.go index 7e24a42b9..ecdd54c84 100644 --- a/src/migration/artifact.go +++ b/src/migration/artifact.go @@ -25,7 +25,7 @@ import ( "github.com/goharbor/harbor/src/pkg/repository" ) -func upgradeData(ctx context.Context) error { +func abstractArtData(ctx context.Context) error { abstractor := art.NewAbstractor() pros, err := project.Mgr.List(ctx) if err != nil { @@ -52,13 +52,13 @@ func upgradeData(ctx context.Context) error { log.Errorf("failed to list artifacts under the repository %s: %v, skip", repo.Name, err) continue } - for _, art := range arts { - if err = abstract(ctx, abstractor, art); err != nil { - log.Errorf("failed to abstract the artifact %s@%s: %v, skip", art.RepositoryName, art.Digest, err) + for _, a := range arts { + if err = abstract(ctx, abstractor, a); err != nil { + log.Errorf("failed to abstract the artifact %s@%s: %v, skip", a.RepositoryName, a.Digest, err) continue } - if err = artifact.Mgr.Update(ctx, art); err != nil { - log.Errorf("failed to update the artifact %s@%s: %v, skip", repo.Name, art.Digest, err) + if err = artifact.Mgr.Update(ctx, a); err != nil { + log.Errorf("failed to update the artifact %s@%s: %v, skip", repo.Name, a.Digest, err) continue } } @@ -67,7 +67,7 @@ func upgradeData(ctx context.Context) error { } // update data version - return setDataVersion(ctx, 30) + return setDataVersion(ctx, dataversionV2_0_0) } func abstract(ctx context.Context, abstractor art.Abstractor, art *artifact.Artifact) error { diff --git a/src/migration/migration.go b/src/migration/migration.go index a4ad69417..d6e6e092b 100644 --- a/src/migration/migration.go +++ b/src/migration/migration.go @@ -25,30 +25,36 @@ import ( "github.com/golang-migrate/migrate/v4" ) -// Migrate the database schema and data -func Migrate(database *models.Database) error { +const ( + schemaVersionV1_10_0 = 15 + // data version for tracking the data integrity in the DB, it can be different from schema version + dataversionV2_0_0 = 30 +) + +// MigrateDB upgrades DB schema and do necessary transformation of the data in DB +func MigrateDB(database *models.Database) error { // check the database schema version migrator, err := dao.NewMigrator(database.PostGreSQL) if err != nil { return err } defer migrator.Close() - schemaVersion, _, err := migrator.Version() if err != nil && err != migrate.ErrNilVersion { return err } log.Debugf("current database schema version: %v", schemaVersion) // prior to 1.9, version = 0 means fresh install - if schemaVersion > 0 && schemaVersion < 10 { - return fmt.Errorf("please upgrade to version 1.9 first") + if schemaVersion > 0 && schemaVersion < schemaVersionV1_10_0 { + return fmt.Errorf("please upgrade to version 1.10 first") } - // update database schema - if err := dao.UpgradeSchema(database); err != nil { - return err - } + return dao.UpgradeSchema(database) +} +// AbstractArtifactData accesses the registry to +func AbstractArtifactData() error { + log.Info("Abstracting artifact data to DB...") ctx := orm.NewContext(context.Background(), beegorm.NewOrm()) dataVersion, err := getDataVersion(ctx) if err != nil { @@ -56,16 +62,21 @@ func Migrate(database *models.Database) error { } log.Debugf("current data version: %v", dataVersion) // the abstract logic already done before, skip - if dataVersion == 30 { - log.Debug("no change in data, skip") + if dataVersion >= dataversionV2_0_0 { + log.Info("No need to abstract artifact data. Skip") return nil } + return abstractArtData(ctx) +} - // upgrade data - if err = upgradeData(ctx); err != nil { +// Migrate the database schema and abstract artifact data +func Migrate(database *models.Database) error { + if err := MigrateDB(database); err != nil { + return err + } + if err := AbstractArtifactData(); err != nil { return err } - return nil }