Bump up the migrate tool of notary

fixes #5863
The migrate binary that we include in notary is quite out dated.
Additionally it introduced a breaking change, more details see #5863

In this commit a go program was added to workaround this issue to ensure the
migration process works, and refined bootstrap scripts and make process accordingly.

Signed-off-by: Daniel Jiang <jiangd@vmware.com>

Makefile

@@ -102,6 +102,7 @@ CLAIRVERSION=v2.0.7
 CLAIRDBVERSION=$(VERSIONTAG)
 MIGRATORVERSION=$(VERSIONTAG)
 REDISVERSION=$(VERSIONTAG)
+NOTARYMIGRATEVERSION=v3.5.4
 
 # version of chartmuseum
 CHARTMUSEUMVERSION=v0.8.1
@@ -132,11 +133,13 @@ GOBUILDPATH_ADMINSERVER=$(GOBUILDPATH)/src/adminserver
 GOBUILDPATH_CORE=$(GOBUILDPATH)/src/core
 GOBUILDPATH_JOBSERVICE=$(GOBUILDPATH)/src/jobservice
 GOBUILDPATH_REGISTRYCTL=$(GOBUILDPATH)/src/registryctl
+GOBUILDPATH_MIGRATEPATCH=$(GOBUILDPATH)/src/cmd/migrate-patch
 GOBUILDMAKEPATH=$(GOBUILDPATH)/make
 GOBUILDMAKEPATH_ADMINSERVER=$(GOBUILDMAKEPATH)/photon/adminserver
 GOBUILDMAKEPATH_CORE=$(GOBUILDMAKEPATH)/photon/core
 GOBUILDMAKEPATH_JOBSERVICE=$(GOBUILDMAKEPATH)/photon/jobservice
 GOBUILDMAKEPATH_REGISTRYCTL=$(GOBUILDMAKEPATH)/photon/registryctl
+GOBUILDMAKEPATH_NOTARY=$(GOBUILDMAKEPATH)/photon/notary
 
 # binary
 ADMINSERVERBINARYPATH=$(MAKEDEVPATH)/adminserver
@@ -147,6 +150,7 @@ JOBSERVICEBINARYPATH=$(MAKEDEVPATH)/jobservice
 JOBSERVICEBINARYNAME=harbor_jobservice
 REGISTRYCTLBINARYPATH=$(MAKEDEVPATH)/registryctl
 REGISTRYCTLBINARYNAME=harbor_registryctl
+MIGRATEPATCHBINARYNAME=migrate-patch
 
 # configfile
 CONFIGPATH=$(MAKEPATH)
@@ -288,7 +292,12 @@ compile_registryctl:
 	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_REGISTRYCTL) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -o $(GOBUILDMAKEPATH_REGISTRYCTL)/$(REGISTRYCTLBINARYNAME)
 	@echo "Done."
 
-compile:check_environment compile_adminserver compile_core compile_jobservice compile_registryctl
+compile_notary_migrate_patch:
+	@echo "compiling binary for migrate patch (golang image)..."
+	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_MIGRATEPATCH) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -o $(GOBUILDMAKEPATH_NOTARY)/$(MIGRATEPATCHBINARYNAME)
+	@echo "Done."
+
+compile:check_environment compile_adminserver compile_core compile_jobservice compile_registryctl compile_notary_migrate_patch
 	
 prepare:
 	@echo "preparing..."
@@ -296,7 +305,7 @@ prepare:
 
 build:
 	make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) \
-	 -e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) \
+	 -e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
 	 -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \
 	 -e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \
 	 -e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER)
@@ -320,7 +329,6 @@ modify_composefile_clair:
 	@cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRTPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
 	@$(SEDCMD) -i -e 's/__postgresql_version__/$(CLAIRDBVERSION)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
 	@$(SEDCMD) -i -e 's/__clair_version__/$(CLAIRVERSION)-$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
-
 modify_composefile_chartmuseum:
 	@echo "preparing docker-compose chartmuseum file..."
 	@cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECHARTMUSEUMTPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECHARTMUSEUMFILENAME)

make/photon/Makefile

@@ -167,13 +167,11 @@ _build_notary:
 	@if [ "$(NOTARYFLAG)" = "true" ] ; then \
 		if [ "$(BUILDBIN)" != "true" ] ; then \
 			rm -rf $(DOCKERFILEPATH_NOTARY)/binary && mkdir -p $(DOCKERFILEPATH_NOTARY)/binary && \
-			$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/release-$(NOTARYVERSION)/notary-signer, $(DOCKERFILEPATH_NOTARY)/binary/notary-signer) && \
+			$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/release-$(NOTARYVERSION)/binary-bundle.tgz, $(DOCKERFILEPATH_NOTARY)/binary-bundle.tgz); \
-			$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/release-$(NOTARYVERSION)/notary-server, $(DOCKERFILEPATH_NOTARY)/binary/notary-server) ; \
+			cd $(DOCKERFILEPATH_NOTARY) && tar -zvxf binary-bundle.tgz && cd - ; \
 		else \
-			cd $(DOCKERFILEPATH_NOTARY) && $(DOCKERFILEPATH_NOTARY)/builder $(NOTARYVERSION) && cd - ; \
+			cd $(DOCKERFILEPATH_NOTARY) && $(DOCKERFILEPATH_NOTARY)/builder $(NOTARYVERSION) $(NOTARYMIGRATEVERSION) && cd - ; \
 		fi ; \
-		$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/release-$(NOTARYVERSION)/notary-migrate-postgresql.tgz, $(DOCKERFILEPATH_NOTARY)/binary/notary-migrate.tgz); \
-		cd $(DOCKERFILEPATH_NOTARY)/binary && tar -zvxf notary-migrate.tgz && cd - ; \
 		echo "building notary container for photon..."; \
 		chmod 655 $(DOCKERFILEPATH_NOTARY)/binary/notary-signer && $(DOCKERBUILD) -f $(DOCKERFILEPATH_NOTARY)/$(DOCKERFILENAME_NOTARYSIGNER) -t $(DOCKERIMAGENAME_NOTARYSIGNER):$(NOTARYVERSION)-$(VERSIONTAG) . ; \
 		chmod 655 $(DOCKERFILEPATH_NOTARY)/binary/notary-server && $(DOCKERBUILD) -f $(DOCKERFILEPATH_NOTARY)/$(DOCKERFILENAME_NOTARYSERVER) -t $(DOCKERIMAGENAME_NOTARYSERVER):$(NOTARYVERSION)-$(VERSIONTAG) . ; \

make/photon/notary/binary.Dockerfile

@@ -1,8 +1,13 @@
 FROM golang:1.11.2
 
+ARG NOTARY_VERSION
+ARG MIGRATE_VERSION
+RUN test -n "$NOTARY_VERSION"
+RUN test -n "$MIGRATE_VERSION"
 ENV NOTARYPKG github.com/theupdateframework/notary
+ENV MIGRATEPKG github.com/golang-migrate/migrate
 
-COPY . /go/src/${NOTARYPKG}
+RUN git clone -b $NOTARY_VERSION https://github.com/theupdateframework/notary.git /go/src/${NOTARYPKG}
 WORKDIR /go/src/${NOTARYPKG}
 
 RUN go install -tags pkcs11 \
@@ -10,3 +15,16 @@ RUN go install -tags pkcs11 \
 
 RUN go install -tags pkcs11 \
     -ldflags "-w -X ${NOTARYPKG}/version.GitCommit=`git rev-parse --short HEAD` -X ${NOTARYPKG}/version.NotaryVersion=`cat NOTARY_VERSION`" ${NOTARYPKG}/cmd/notary-signer
+RUN cp -r /go/src/${NOTARYPKG}/migrations/ / 
+
+RUN git clone -b $MIGRATE_VERSION https://github.com/golang-migrate/migrate /go/src/${MIGRATEPKG}
+WORKDIR /go/src/${MIGRATEPKG}
+
+RUN curl -fsSL -o /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 && chmod +x /usr/local/bin/dep
+RUN dep ensure -vendor-only
+
+ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse"
+ENV SOURCES="file go_bindata github aws_s3 google_cloud_storage"
+
+RUN go install -tags "$DATABASES $SOURCES" -ldflags="-X main.Version=${MIGRATE_VERSION}" ${MIGRATEPKG}/cli && mv /go/bin/cli /go/bin/migrate
+

make/photon/notary/builder

@@ -2,43 +2,28 @@
 
 set +e
 
-if [ -z $1 ]; then
+if [ -z $2 ]; then
-  error "Please set the 'version' variable"
+  error "Please set the notary and migrate version"
   exit 1
 fi
 
-VERSION="$1"
+echo "Building notary and golang-migrate from source, notary version: $1, golang-migrate version: $2"
-
 set -e
 
 # the temp folder to store binary file...
 mkdir -p binary
-rm -rf binary/notary-server || true
+rm -rf binary/* || true
-rm -rf binary/notary-signer || true
 
 cd `dirname $0`
-cur=$PWD
+docker build --build-arg NOTARY_VERSION=$1 --build-arg MIGRATE_VERSION=$2 -f ./binary.Dockerfile -t notary-binary .
 
-# the temp folder to store notary source code...
+echo 'copy the binary files to local...'
-TEMP=`mktemp -d /$TMPDIR/notary.XXXXXX`
+ID=$(docker create notary-binary)
-git clone -b $VERSION https://github.com/theupdateframework/notary.git $TEMP
+docker cp $ID:/go/bin/notary-server binary/
-
+docker cp $ID:/go/bin/notary-signer binary/
-echo 'build the notary binary bases on the golang:1.11.2...'
+docker cp $ID:/go/bin/migrate binary/
-cp binary.Dockerfile $TEMP 
+docker cp $ID:/migrations binary/
-cd $TEMP
-docker build -f binary.Dockerfile -t notary-golang $TEMP
-cp -r $TEMP/migrations binary
-
-echo 'copy the notary binary to local...'
-ID=$(docker create notary-golang)
-echo $ID
-cd $cur
-docker cp $ID:/go/bin/notary-server binary
-docker cp $ID:/go/bin/notary-signer binary
 
 docker rm -f $ID
-docker rmi -f notary-golang
+docker rmi -f notary-binary
-
-rm -rf $TEMP
-
 

make/photon/notary/server-start.sh

@@ -1,2 +1,2 @@
 #!/bin/sh
-sudo -E -u \#10000 sh -c "/usr/bin/env /migrations/migrate.sh && /bin/notary-server -config=/etc/notary/server-config.postgres.json -logf=logfmt"
+sudo -E -u \#10000 sh -c "migrate-patch -database='${DB_URL}' && /migrations/migrate.sh && /bin/notary-server -config=/etc/notary/server-config.postgres.json -logf=logfmt"

make/photon/notary/server.Dockerfile

@@ -5,10 +5,11 @@ RUN tdnf install -y shadow sudo \
     && groupadd -r -g 10000 notary \
     && useradd --no-log-init -r -g 10000 -u 10000 notary
 
+COPY ./make/photon/notary/migrate-patch /bin/migrate-patch
 COPY ./make/photon/notary/binary/notary-server /bin/notary-server
 COPY ./make/photon/notary/binary/migrate /bin/migrate
 COPY ./make/photon/notary/binary/migrations/ /migrations/
 COPY ./make/photon/notary/server-start.sh /bin/server-start.sh
-RUN chmod u+x /bin/notary-server /migrations/migrate.sh /bin/migrate /bin/server-start.sh
+RUN chmod +x /bin/notary-server /migrations/migrate.sh /bin/migrate /bin/migrate-patch /bin/server-start.sh
 ENV SERVICE_NAME=notary_server
 ENTRYPOINT [ "/bin/server-start.sh" ]

make/photon/notary/signer-start.sh

@@ -1,2 +1,2 @@
 #!/bin/sh
-sudo -E -u \#10000 sh -c "/usr/bin/env && /migrations/migrate.sh && /bin/notary-signer -config=/etc/notary/signer-config.postgres.json -logf=logfmt"
+sudo -E -u \#10000 sh -c "migrate-patch -database='${DB_URL}' && /migrations/migrate.sh && /bin/notary-signer -config=/etc/notary/signer-config.postgres.json -logf=logfmt"

make/photon/notary/signer.Dockerfile

@@ -4,11 +4,12 @@ RUN tdnf install -y shadow sudo \
     && tdnf clean all \
     && groupadd -r -g 10000 notary \
     && useradd --no-log-init -r -g 10000 -u 10000 notary
+COPY ./make/photon/notary/migrate-patch /bin/migrate-patch
 COPY ./make/photon/notary/binary/notary-signer /bin/notary-signer
 COPY ./make/photon/notary/binary/migrate /bin/migrate
 COPY ./make/photon/notary/binary/migrations/ /migrations/
 COPY ./make/photon/notary/signer-start.sh /bin/signer-start.sh
 
-RUN chmod u+x /bin/notary-signer /migrations/migrate.sh /bin/migrate /bin/signer-start.sh
+RUN chmod +x /bin/notary-signer /migrations/migrate.sh /bin/migrate /bin/migrate-patch /bin/signer-start.sh
 ENV SERVICE_NAME=notary_signer
 ENTRYPOINT [ "/bin/signer-start.sh" ]

src/cmd/migrate-patch/README.md

@@ -0,0 +1,6 @@
+# Migrate Patch
+This is a simple program to fix the breakage that was introduced by migrate in notary.
+## Usage
+```sh
+patch -database <db_url>
+```

src/cmd/migrate-patch/main.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"database/sql"
+	"flag"
+	_ "github.com/lib/pq"
+	"log"
+	"strings"
+	"time"
+)
+
+var dbURL string
+
+const pgSQLAlterStmt string = `ALTER TABLE schema_migrations ADD COLUMN "dirty" boolean NOT NULL DEFAULT false`
+const pgSQLCheckColStmt string = `SELECT T1.C1, T2.C2 FROM
+(SELECT COUNT(*) AS C1 FROM information_schema.tables WHERE table_name='schema_migrations') T1,
+(SELECT COUNT(*) AS C2 FROM information_schema.columns WHERE table_name='schema_migrations' and column_name='dirty') T2`
+const pgSQLDelRows string = `DELETE FROM schema_migrations t WHERE t.version < ( SELECT MAX(version) FROM schema_migrations )`
+
+func init() {
+	urlUsage := `The URL to the target database (driver://url).  Currently it only supports postgres`
+	flag.StringVar(&dbURL, "database", "", urlUsage)
+}
+
+func main() {
+	flag.Parse()
+	log.Printf("Updating database.")
+	if !strings.HasPrefix(dbURL, "postgres://") {
+		log.Fatalf("Invalid URL: '%s'\n", dbURL)
+	}
+	db, err := sql.Open("postgres", dbURL)
+	if err != nil {
+		log.Fatalf("Failed to connect to Database, error: %v\n", err)
+	}
+	defer db.Close()
+	c := make(chan int, 1)
+	go func() {
+		err := db.Ping()
+		for ; err != nil; err = db.Ping() {
+			log.Printf("Failed to Ping DB, sleep for 1 second.\n")
+			time.Sleep(1 * time.Second)
+		}
+		c <- 1
+	}()
+	select {
+	case <-c:
+	case <-time.After(30 * time.Second):
+		log.Fatal("Failed to connect DB after 30 seconds, time out. \n")
+
+	}
+	row := db.QueryRow(pgSQLCheckColStmt)
+	var tblCount, colCount int
+	if err := row.Scan(&tblCount, &colCount); err != nil {
+		log.Fatalf("Failed to check schema_migrations table, error: %v \n", err)
+	}
+	if tblCount == 0 {
+		log.Printf("schema_migrations table does not exist, skip.\n")
+		return
+	}
+	if colCount > 0 {
+		log.Printf("schema_migrations table does not require update, skip.\n")
+		return
+	}
+	if _, err := db.Exec(pgSQLDelRows); err != nil {
+		log.Fatalf("Failed to clean up table, error: %v", err)
+	}
+	if _, err := db.Exec(pgSQLAlterStmt); err != nil {
+		log.Fatalf("Failed to update database, error: %v \n", err)
+	}
+	log.Printf("Done updating database. \n")
+}