Merge pull request #6983 from reasonerjt/new-bump-up-notary-migrate

Bump up the migrate tool of notary
This commit is contained in:
Wenkai Yin 2019-02-21 18:03:17 +08:00 committed by GitHub
commit a3e4772fe9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 128 additions and 40 deletions

View File

@ -102,6 +102,7 @@ CLAIRVERSION=v2.0.7
CLAIRDBVERSION=$(VERSIONTAG)
MIGRATORVERSION=$(VERSIONTAG)
REDISVERSION=$(VERSIONTAG)
NOTARYMIGRATEVERSION=v3.5.4
# version of chartmuseum
CHARTMUSEUMVERSION=v0.8.1
@ -132,11 +133,13 @@ GOBUILDPATH_ADMINSERVER=$(GOBUILDPATH)/src/adminserver
GOBUILDPATH_CORE=$(GOBUILDPATH)/src/core
GOBUILDPATH_JOBSERVICE=$(GOBUILDPATH)/src/jobservice
GOBUILDPATH_REGISTRYCTL=$(GOBUILDPATH)/src/registryctl
GOBUILDPATH_MIGRATEPATCH=$(GOBUILDPATH)/src/cmd/migrate-patch
GOBUILDMAKEPATH=$(GOBUILDPATH)/make
GOBUILDMAKEPATH_ADMINSERVER=$(GOBUILDMAKEPATH)/photon/adminserver
GOBUILDMAKEPATH_CORE=$(GOBUILDMAKEPATH)/photon/core
GOBUILDMAKEPATH_JOBSERVICE=$(GOBUILDMAKEPATH)/photon/jobservice
GOBUILDMAKEPATH_REGISTRYCTL=$(GOBUILDMAKEPATH)/photon/registryctl
GOBUILDMAKEPATH_NOTARY=$(GOBUILDMAKEPATH)/photon/notary
# binary
ADMINSERVERBINARYPATH=$(MAKEDEVPATH)/adminserver
@ -147,6 +150,7 @@ JOBSERVICEBINARYPATH=$(MAKEDEVPATH)/jobservice
JOBSERVICEBINARYNAME=harbor_jobservice
REGISTRYCTLBINARYPATH=$(MAKEDEVPATH)/registryctl
REGISTRYCTLBINARYNAME=harbor_registryctl
MIGRATEPATCHBINARYNAME=migrate-patch
# configfile
CONFIGPATH=$(MAKEPATH)
@ -288,7 +292,12 @@ compile_registryctl:
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_REGISTRYCTL) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -o $(GOBUILDMAKEPATH_REGISTRYCTL)/$(REGISTRYCTLBINARYNAME)
@echo "Done."
compile:check_environment compile_adminserver compile_core compile_jobservice compile_registryctl
compile_notary_migrate_patch:
@echo "compiling binary for migrate patch (golang image)..."
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_MIGRATEPATCH) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -o $(GOBUILDMAKEPATH_NOTARY)/$(MIGRATEPATCHBINARYNAME)
@echo "Done."
compile:check_environment compile_adminserver compile_core compile_jobservice compile_registryctl compile_notary_migrate_patch
prepare:
@echo "preparing..."
@ -296,7 +305,7 @@ prepare:
build:
make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) \
-e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) \
-e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
-e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \
-e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \
-e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER)
@ -320,7 +329,6 @@ modify_composefile_clair:
@cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRTPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
@$(SEDCMD) -i -e 's/__postgresql_version__/$(CLAIRDBVERSION)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
@$(SEDCMD) -i -e 's/__clair_version__/$(CLAIRVERSION)-$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
modify_composefile_chartmuseum:
@echo "preparing docker-compose chartmuseum file..."
@cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECHARTMUSEUMTPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECHARTMUSEUMFILENAME)

View File

@ -167,13 +167,11 @@ _build_notary:
@if [ "$(NOTARYFLAG)" = "true" ] ; then \
if [ "$(BUILDBIN)" != "true" ] ; then \
rm -rf $(DOCKERFILEPATH_NOTARY)/binary && mkdir -p $(DOCKERFILEPATH_NOTARY)/binary && \
$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/release-$(NOTARYVERSION)/notary-signer, $(DOCKERFILEPATH_NOTARY)/binary/notary-signer) && \
$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/release-$(NOTARYVERSION)/notary-server, $(DOCKERFILEPATH_NOTARY)/binary/notary-server) ; \
$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/release-$(NOTARYVERSION)/binary-bundle.tgz, $(DOCKERFILEPATH_NOTARY)/binary-bundle.tgz); \
cd $(DOCKERFILEPATH_NOTARY) && tar -zvxf binary-bundle.tgz && cd - ; \
else \
cd $(DOCKERFILEPATH_NOTARY) && $(DOCKERFILEPATH_NOTARY)/builder $(NOTARYVERSION) && cd - ; \
cd $(DOCKERFILEPATH_NOTARY) && $(DOCKERFILEPATH_NOTARY)/builder $(NOTARYVERSION) $(NOTARYMIGRATEVERSION) && cd - ; \
fi ; \
$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/release-$(NOTARYVERSION)/notary-migrate-postgresql.tgz, $(DOCKERFILEPATH_NOTARY)/binary/notary-migrate.tgz); \
cd $(DOCKERFILEPATH_NOTARY)/binary && tar -zvxf notary-migrate.tgz && cd - ; \
echo "building notary container for photon..."; \
chmod 655 $(DOCKERFILEPATH_NOTARY)/binary/notary-signer && $(DOCKERBUILD) -f $(DOCKERFILEPATH_NOTARY)/$(DOCKERFILENAME_NOTARYSIGNER) -t $(DOCKERIMAGENAME_NOTARYSIGNER):$(NOTARYVERSION)-$(VERSIONTAG) . ; \
chmod 655 $(DOCKERFILEPATH_NOTARY)/binary/notary-server && $(DOCKERBUILD) -f $(DOCKERFILEPATH_NOTARY)/$(DOCKERFILENAME_NOTARYSERVER) -t $(DOCKERIMAGENAME_NOTARYSERVER):$(NOTARYVERSION)-$(VERSIONTAG) . ; \

View File

@ -1,8 +1,13 @@
FROM golang:1.11.2
ARG NOTARY_VERSION
ARG MIGRATE_VERSION
RUN test -n "$NOTARY_VERSION"
RUN test -n "$MIGRATE_VERSION"
ENV NOTARYPKG github.com/theupdateframework/notary
ENV MIGRATEPKG github.com/golang-migrate/migrate
COPY . /go/src/${NOTARYPKG}
RUN git clone -b $NOTARY_VERSION https://github.com/theupdateframework/notary.git /go/src/${NOTARYPKG}
WORKDIR /go/src/${NOTARYPKG}
RUN go install -tags pkcs11 \
@ -10,3 +15,16 @@ RUN go install -tags pkcs11 \
RUN go install -tags pkcs11 \
-ldflags "-w -X ${NOTARYPKG}/version.GitCommit=`git rev-parse --short HEAD` -X ${NOTARYPKG}/version.NotaryVersion=`cat NOTARY_VERSION`" ${NOTARYPKG}/cmd/notary-signer
RUN cp -r /go/src/${NOTARYPKG}/migrations/ /
RUN git clone -b $MIGRATE_VERSION https://github.com/golang-migrate/migrate /go/src/${MIGRATEPKG}
WORKDIR /go/src/${MIGRATEPKG}
RUN curl -fsSL -o /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 && chmod +x /usr/local/bin/dep
RUN dep ensure -vendor-only
ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse"
ENV SOURCES="file go_bindata github aws_s3 google_cloud_storage"
RUN go install -tags "$DATABASES $SOURCES" -ldflags="-X main.Version=${MIGRATE_VERSION}" ${MIGRATEPKG}/cli && mv /go/bin/cli /go/bin/migrate

View File

@ -2,43 +2,28 @@
set +e
if [ -z $1 ]; then
error "Please set the 'version' variable"
if [ -z $2 ]; then
error "Please set the notary and migrate version"
exit 1
fi
VERSION="$1"
echo "Building notary and golang-migrate from source, notary version: $1, golang-migrate version: $2"
set -e
# the temp folder to store binary file...
mkdir -p binary
rm -rf binary/notary-server || true
rm -rf binary/notary-signer || true
rm -rf binary/* || true
cd `dirname $0`
cur=$PWD
docker build --build-arg NOTARY_VERSION=$1 --build-arg MIGRATE_VERSION=$2 -f ./binary.Dockerfile -t notary-binary .
# the temp folder to store notary source code...
TEMP=`mktemp -d /$TMPDIR/notary.XXXXXX`
git clone -b $VERSION https://github.com/theupdateframework/notary.git $TEMP
echo 'build the notary binary bases on the golang:1.11.2...'
cp binary.Dockerfile $TEMP
cd $TEMP
docker build -f binary.Dockerfile -t notary-golang $TEMP
cp -r $TEMP/migrations binary
echo 'copy the notary binary to local...'
ID=$(docker create notary-golang)
echo $ID
cd $cur
docker cp $ID:/go/bin/notary-server binary
docker cp $ID:/go/bin/notary-signer binary
echo 'copy the binary files to local...'
ID=$(docker create notary-binary)
docker cp $ID:/go/bin/notary-server binary/
docker cp $ID:/go/bin/notary-signer binary/
docker cp $ID:/go/bin/migrate binary/
docker cp $ID:/migrations binary/
docker rm -f $ID
docker rmi -f notary-golang
rm -rf $TEMP
docker rmi -f notary-binary

View File

@ -1,2 +1,2 @@
#!/bin/sh
sudo -E -u \#10000 sh -c "/usr/bin/env /migrations/migrate.sh && /bin/notary-server -config=/etc/notary/server-config.postgres.json -logf=logfmt"
sudo -E -u \#10000 sh -c "migrate-patch -database='${DB_URL}' && /migrations/migrate.sh && /bin/notary-server -config=/etc/notary/server-config.postgres.json -logf=logfmt"

View File

@ -5,10 +5,11 @@ RUN tdnf install -y shadow sudo \
&& groupadd -r -g 10000 notary \
&& useradd --no-log-init -r -g 10000 -u 10000 notary
COPY ./make/photon/notary/migrate-patch /bin/migrate-patch
COPY ./make/photon/notary/binary/notary-server /bin/notary-server
COPY ./make/photon/notary/binary/migrate /bin/migrate
COPY ./make/photon/notary/binary/migrations/ /migrations/
COPY ./make/photon/notary/server-start.sh /bin/server-start.sh
RUN chmod u+x /bin/notary-server /migrations/migrate.sh /bin/migrate /bin/server-start.sh
RUN chmod +x /bin/notary-server /migrations/migrate.sh /bin/migrate /bin/migrate-patch /bin/server-start.sh
ENV SERVICE_NAME=notary_server
ENTRYPOINT [ "/bin/server-start.sh" ]

View File

@ -1,2 +1,2 @@
#!/bin/sh
sudo -E -u \#10000 sh -c "/usr/bin/env && /migrations/migrate.sh && /bin/notary-signer -config=/etc/notary/signer-config.postgres.json -logf=logfmt"
sudo -E -u \#10000 sh -c "migrate-patch -database='${DB_URL}' && /migrations/migrate.sh && /bin/notary-signer -config=/etc/notary/signer-config.postgres.json -logf=logfmt"

View File

@ -4,11 +4,12 @@ RUN tdnf install -y shadow sudo \
&& tdnf clean all \
&& groupadd -r -g 10000 notary \
&& useradd --no-log-init -r -g 10000 -u 10000 notary
COPY ./make/photon/notary/migrate-patch /bin/migrate-patch
COPY ./make/photon/notary/binary/notary-signer /bin/notary-signer
COPY ./make/photon/notary/binary/migrate /bin/migrate
COPY ./make/photon/notary/binary/migrations/ /migrations/
COPY ./make/photon/notary/signer-start.sh /bin/signer-start.sh
RUN chmod u+x /bin/notary-signer /migrations/migrate.sh /bin/migrate /bin/signer-start.sh
RUN chmod +x /bin/notary-signer /migrations/migrate.sh /bin/migrate /bin/migrate-patch /bin/signer-start.sh
ENV SERVICE_NAME=notary_signer
ENTRYPOINT [ "/bin/signer-start.sh" ]

View File

@ -0,0 +1,6 @@
# Migrate Patch
This is a simple program to fix the breakage that was introduced by migrate in notary.
## Usage
```sh
patch -database <db_url>
```

View File

@ -0,0 +1,71 @@
package main
import (
"database/sql"
"flag"
_ "github.com/lib/pq"
"log"
"strings"
"time"
)
var dbURL string
const pgSQLAlterStmt string = `ALTER TABLE schema_migrations ADD COLUMN "dirty" boolean NOT NULL DEFAULT false`
const pgSQLCheckColStmt string = `SELECT T1.C1, T2.C2 FROM
(SELECT COUNT(*) AS C1 FROM information_schema.tables WHERE table_name='schema_migrations') T1,
(SELECT COUNT(*) AS C2 FROM information_schema.columns WHERE table_name='schema_migrations' and column_name='dirty') T2`
const pgSQLDelRows string = `DELETE FROM schema_migrations t WHERE t.version < ( SELECT MAX(version) FROM schema_migrations )`
func init() {
urlUsage := `The URL to the target database (driver://url). Currently it only supports postgres`
flag.StringVar(&dbURL, "database", "", urlUsage)
}
func main() {
flag.Parse()
log.Printf("Updating database.")
if !strings.HasPrefix(dbURL, "postgres://") {
log.Fatalf("Invalid URL: '%s'\n", dbURL)
}
db, err := sql.Open("postgres", dbURL)
if err != nil {
log.Fatalf("Failed to connect to Database, error: %v\n", err)
}
defer db.Close()
c := make(chan int, 1)
go func() {
err := db.Ping()
for ; err != nil; err = db.Ping() {
log.Printf("Failed to Ping DB, sleep for 1 second.\n")
time.Sleep(1 * time.Second)
}
c <- 1
}()
select {
case <-c:
case <-time.After(30 * time.Second):
log.Fatal("Failed to connect DB after 30 seconds, time out. \n")
}
row := db.QueryRow(pgSQLCheckColStmt)
var tblCount, colCount int
if err := row.Scan(&tblCount, &colCount); err != nil {
log.Fatalf("Failed to check schema_migrations table, error: %v \n", err)
}
if tblCount == 0 {
log.Printf("schema_migrations table does not exist, skip.\n")
return
}
if colCount > 0 {
log.Printf("schema_migrations table does not require update, skip.\n")
return
}
if _, err := db.Exec(pgSQLDelRows); err != nil {
log.Fatalf("Failed to clean up table, error: %v", err)
}
if _, err := db.Exec(pgSQLAlterStmt); err != nil {
log.Fatalf("Failed to update database, error: %v \n", err)
}
log.Printf("Done updating database. \n")
}