Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Provide a standalone migrator to migrate DB schema. 

Fixes #11885
This part will not by default be packaged into release.
A README.md will be added in another commit.

Signed-off-by: Daniel Jiang <jiangd@vmware.com>
This commit is contained in:
Daniel Jiang 2020-07-13 16:57:09 +08:00
parent dc58271d3e
commit 4f94f59d2a
11 changed files with 189 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Makefile
View File

@ -177,11 +177,13 @@ GOBUILDPATH_CORE=$(GOBUILDPATHINCONTAINER)/src/core
GOBUILDPATH_JOBSERVICE=$(GOBUILDPATHINCONTAINER)/src/jobservice
GOBUILDPATH_REGISTRYCTL=$(GOBUILDPATHINCONTAINER)/src/registryctl
GOBUILDPATH_MIGRATEPATCH=$(GOBUILDPATHINCONTAINER)/src/cmd/migrate-patch
GOBUILDPATH_STANDALONE_DB_MIGRATOR=$(GOBUILDPATHINCONTAINER)/src/cmd/standalone-db-migrator
GOBUILDMAKEPATH=make
GOBUILDMAKEPATH_CORE=$(GOBUILDMAKEPATH)/photon/core
GOBUILDMAKEPATH_JOBSERVICE=$(GOBUILDMAKEPATH)/photon/jobservice
GOBUILDMAKEPATH_REGISTRYCTL=$(GOBUILDMAKEPATH)/photon/registryctl
GOBUILDMAKEPATH_NOTARY=$(GOBUILDMAKEPATH)/photon/notary
GOBUILDMAKEPATH_STANDALONE_DB_MIGRATOR=$(GOBUILDMAKEPATH)/photon/standalone-db-migrator
# binary
CORE_BINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_CORE)
@ -192,6 +194,8 @@ REGISTRYCTLBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_REGISTRYCTL)
REGISTRYCTLBINARYNAME=harbor_registryctl
MIGRATEPATCHBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_NOTARY)
MIGRATEPATCHBINARYNAME=migrate-patch
STANDALONE_DB_MIGRATOR_BINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_STANDALONE_DB_MIGRATOR)
STANDALONE_DB_MIGRATOR_BINARYNAME=migrate
# configfile
CONFIGPATH=$(MAKEPATH)
@ -378,6 +382,11 @@ compile_notary_migrate_patch:
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_MIGRATEPATCH) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_NOTARY)/$(MIGRATEPATCHBINARYNAME)
@echo "Done."
compile_standalone_db_migrator:
@echo "compiling binary for standalone db migrator (golang image)..."
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_STANDALONE_DB_MIGRATOR) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_STANDALONE_DB_MIGRATOR)/$(STANDALONE_DB_MIGRATOR_BINARYNAME)
@echo "Done."
compile: check_environment versions_prepare compile_core compile_jobservice compile_registryctl compile_notary_migrate_patch
update_prepare_version:
@ -406,6 +415,9 @@ build:
-e CLAIRURL=$(CLAIRURL) -e CHARTURL=$(CHARTURL) -e NORARYURL=$(NORARYURL) -e REGISTRYURL=$(REGISTRYURL) -e CLAIR_ADAPTER_DOWNLOAD_URL=$(CLAIR_ADAPTER_DOWNLOAD_URL) \
-e TRIVY_DOWNLOAD_URL=$(TRIVY_DOWNLOAD_URL) -e TRIVY_ADAPTER_DOWNLOAD_URL=$(TRIVY_ADAPTER_DOWNLOAD_URL)
build_standalone_db_migrator: compile_standalone_db_migrator
make -f $(MAKEFILEPATH_PHOTON)/Makefile _build_standalone_db_migrator -e BASEIMAGETAG=$(BASEIMAGETAG) -e VERSIONTAG=$(VERSIONTAG)
build_base_docker:
@for name in chartserver clair clair-adapter trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
echo $$name ; \

10
make/photon/Makefile
View File

@ -95,6 +95,11 @@ DOCKERFILEPATH_REDIS=$(DOCKERFILEPATH)/redis
DOCKERFILENAME_REDIS=Dockerfile
DOCKERIMAGENAME_REDIS=goharbor/redis-photon
DOCKERFILEPATH_STANDALONE_DB_MIGRATOR=$(DOCKERFILEPATH)/standalone-db-migrator
DOCKERFILENAME_STANDALONE_DB_MIGRATOR=Dockerfile
DOCKERIMAGENAME_STANDALONE_DB_MIGRATOR=goharbor/standalone-db-migrator
# for chart server (chartmuseum)
DOCKERFILEPATH_CHART_SERVER=$(DOCKERFILEPATH)/chartserver
DOCKERFILENAME_CHART_SERVER=Dockerfile
@ -243,6 +248,11 @@ _build_redis:
@$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_REDIS)/$(DOCKERFILENAME_REDIS) -t $(DOCKERIMAGENAME_REDIS):$(VERSIONTAG) .
@echo "Done."
_build_standalone_db_migrator:
@echo "building standalone db migrator image for photon..."
@$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_STANDALONE_DB_MIGRATOR)/$(DOCKERFILENAME_STANDALONE_DB_MIGRATOR) -t $(DOCKERIMAGENAME_STANDALONE_DB_MIGRATOR):$(VERSIONTAG) .
@echo "Done."
define _extract_archive
echo "download $1";\
$(CURL) --connect-timeout 30 -f -k -L $1 | tar xvz -C $2 || exit 1

17
make/photon/standalone-db-migrator/Dockerfile Normal file
View File

@ -0,0 +1,17 @@
ARG harbor_base_image_version
ARG harbor_base_namespace
FROM ${harbor_base_namespace}/harbor-db-base:${harbor_base_image_version}
ENV EXTERNAL_DB 0
RUN mkdir /harbor/
COPY ./make/migrations /migrations
COPY ./make/photon/standalone-db-migrator/migrate /harbor/
COPY ./make/photon/standalone-db-migrator/entrypoint.sh /harbor/
RUN chown -R postgres:postgres /harbor/ \
&& chown -R postgres:postgres /migrations/ \
&& chmod u+x /harbor/migrate /harbor/entrypoint.sh
USER postgres
ENTRYPOINT ["/harbor/entrypoint.sh"]

6
make/photon/standalone-db-migrator/entrypoint.sh Normal file
View File

@ -0,0 +1,6 @@
#!/bin/bash
set -e
[ $EXTERNAL_DB -eq 1 ] || pg_ctl start -w -t 60 -D ${PGDATA}
/harbor/migrate
[ $EXTERNAL_DB -eq 1 ] || pg_ctl stop -D ${PGDATA}

58
src/cmd/standalone-db-migrator/main.go Normal file
View File

@ -0,0 +1,58 @@
package main
import (
"os"
"strconv"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/migration"
)
// key: env var, value: default value
var defaultAttrs = map[string]string{
"POSTGRESQL_HOST": "localhost",
"POSTGRESQL_PORT": "5432",
"POSTGRESQL_USERNAME": "postgres",
"POSTGRESQL_PASSWORD": "password",
"POSTGRESQL_DATABASE": "registry",
"POSTGRESQL_SSLMODE": "disable",
}
func main() {
p, _ := strconv.Atoi(getAttr("POSTGRESQL_PORT"))
db := &models.Database{
Type: "postgresql",
PostGreSQL: &models.PostGreSQL{
Host: getAttr("POSTGRESQL_HOST"),
Port: p,
Username: getAttr("POSTGRESQL_USERNAME"),
Password: getAttr("POSTGRESQL_PASSWORD"),
Database: getAttr("POSTGRESQL_DATABASE"),
SSLMode: getAttr("POSTGRESQL_SSLMODE"),
MaxIdleConns: 5,
MaxOpenConns: 5,
},
}
log.Info("Migrating the data to latest schema...")
log.Infof("DB info: postgres://%s@%s:%d/%s?sslmode=%s", db.PostGreSQL.Username, db.PostGreSQL.Host,
db.PostGreSQL.Port, db.PostGreSQL.Database, db.PostGreSQL.SSLMode)
if err := dao.InitDatabase(db); err != nil {
log.Fatalf("failed to initialize database: %v", err)
}
if err := migration.MigrateDB(db); err != nil {
log.Fatalf("failed to migrate DB: %v", err)
}
log.Info("Migration done. The data schema in DB is now update to date.")
}
func getAttr(k string) string {
v := os.Getenv(k)
if len(v) > 0 {
return v
}
return defaultAttrs[k]
}

21
src/common/dao/base.go
View File

@ -153,3 +153,24 @@ func Escape(str string) string {
str = strings.Replace(str, `_`, `\_`, -1)
return str
}
// implements github.com/golang-migrate/migrate/v4.Logger
type mLogger struct {
logger *log.Logger
}
func newMigrateLogger() *mLogger {
return &mLogger{
logger: log.DefaultLogger().WithDepth(5),
}
}
// Verbose ...
func (l *mLogger) Verbose() bool {
return l.logger.GetLevel() <= log.DebugLevel
}
// Printf ...
func (l *mLogger) Printf(format string, v ...interface{}) {
l.logger.Infof(format, v...)
}

17
src/common/dao/base_test.go Normal file
View File

@ -0,0 +1,17 @@
package dao
import (
"testing"
"github.com/goharbor/harbor/src/lib/log"
"github.com/stretchr/testify/assert"
)
func TestMLogger_Verbose(t *testing.T) {
l := newMigrateLogger()
if log.DefaultLogger().GetLevel() <= log.DebugLevel {
assert.True(t, l.Verbose())
} else {
assert.False(t, l.Verbose())
}
}

7
src/common/dao/pgsql.go
View File

@ -151,5 +151,10 @@ func NewMigrator(database *models.PostGreSQL) (*migrate.Migrate, error) {
path = defaultMigrationPath
}
srcURL := fmt.Sprintf("file://%s", path)
return migrate.New(srcURL, dbURL.String())
m, err := migrate.New(srcURL, dbURL.String())
if err != nil {
return nil, err
}
m.Log = newMigrateLogger()
return m, nil
}

10
src/lib/log/logger.go
View File

@ -261,6 +261,11 @@ func (l *Logger) Fatalf(format string, v ...interface{}) {
os.Exit(1)
}
// GetLevel returns the verbosity level of this logger
func (l *Logger) GetLevel() Level {
return l.lvl
}
func (l *Logger) getLine() string {
var str string
if !l.skipLine {
@ -326,6 +331,11 @@ func Fatalf(format string, v ...interface{}) {
logger.WithDepth(4).Fatalf(format, v...)
}
// GetLevel return the verbosity level of default logger
func GetLevel() Level {
return logger.GetLevel()
}
func line(callDepth int) string {
_, file, line, ok := runtime.Caller(callDepth)
if !ok {

14
src/migration/artifact.go
View File

@ -25,7 +25,7 @@ import (
"github.com/goharbor/harbor/src/pkg/repository"
)
func upgradeData(ctx context.Context) error {
func abstractArtData(ctx context.Context) error {
abstractor := art.NewAbstractor()
pros, err := project.Mgr.List(ctx)
if err != nil {
@ -52,13 +52,13 @@ func upgradeData(ctx context.Context) error {
log.Errorf("failed to list artifacts under the repository %s: %v, skip", repo.Name, err)
continue
}
for _, art := range arts {
if err = abstract(ctx, abstractor, art); err != nil {
log.Errorf("failed to abstract the artifact %s@%s: %v, skip", art.RepositoryName, art.Digest, err)
for _, a := range arts {
if err = abstract(ctx, abstractor, a); err != nil {
log.Errorf("failed to abstract the artifact %s@%s: %v, skip", a.RepositoryName, a.Digest, err)
continue
}
if err = artifact.Mgr.Update(ctx, art); err != nil {
log.Errorf("failed to update the artifact %s@%s: %v, skip", repo.Name, art.Digest, err)
if err = artifact.Mgr.Update(ctx, a); err != nil {
log.Errorf("failed to update the artifact %s@%s: %v, skip", repo.Name, a.Digest, err)
continue
}
}
@ -67,7 +67,7 @@ func upgradeData(ctx context.Context) error {
}
// update data version
return setDataVersion(ctx, 30)
return setDataVersion(ctx, dataversionV2_0_0)
}
func abstract(ctx context.Context, abstractor art.Abstractor, art *artifact.Artifact) error {

39
src/migration/migration.go
View File

@ -25,30 +25,36 @@ import (
"github.com/golang-migrate/migrate/v4"
)
// Migrate the database schema and data
func Migrate(database *models.Database) error {
const (
schemaVersionV1_10_0 = 15
// data version for tracking the data integrity in the DB, it can be different from schema version
dataversionV2_0_0 = 30
)
// MigrateDB upgrades DB schema and do necessary transformation of the data in DB
func MigrateDB(database *models.Database) error {
// check the database schema version
migrator, err := dao.NewMigrator(database.PostGreSQL)
if err != nil {
return err
}
defer migrator.Close()
schemaVersion, _, err := migrator.Version()
if err != nil && err != migrate.ErrNilVersion {
return err
}
log.Debugf("current database schema version: %v", schemaVersion)
// prior to 1.9, version = 0 means fresh install
if schemaVersion > 0 && schemaVersion < 10 {
return fmt.Errorf("please upgrade to version 1.9 first")
if schemaVersion > 0 && schemaVersion < schemaVersionV1_10_0 {
return fmt.Errorf("please upgrade to version 1.10 first")
}
// update database schema
if err := dao.UpgradeSchema(database); err != nil {
return err
}
return dao.UpgradeSchema(database)
}
// AbstractArtifactData accesses the registry to
func AbstractArtifactData() error {
log.Info("Abstracting artifact data to DB...")
ctx := orm.NewContext(context.Background(), beegorm.NewOrm())
dataVersion, err := getDataVersion(ctx)
if err != nil {
@ -56,16 +62,21 @@ func Migrate(database *models.Database) error {
}
log.Debugf("current data version: %v", dataVersion)
// the abstract logic already done before, skip
if dataVersion == 30 {
log.Debug("no change in data, skip")
if dataVersion >= dataversionV2_0_0 {
log.Info("No need to abstract artifact data. Skip")
return nil
}
return abstractArtData(ctx)
}
// upgrade data
if err = upgradeData(ctx); err != nil {
// Migrate the database schema and abstract artifact data
func Migrate(database *models.Database) error {
if err := MigrateDB(database); err != nil {
return err
}
if err := AbstractArtifactData(); err != nil {
return err
}
return nil
}