mirror of
https://github.com/goharbor/harbor.git
synced 2024-11-25 03:35:21 +01:00
Bump up the migrate tool of notary
fixes #5863 The migrate binary that we include in notary is quite out dated. Additionally it introduced a breaking change, more details see #5863 In this commit a go program was added to workaround this issue to ensure the migration process works, and refined bootstrap scripts and make process accordingly. Signed-off-by: Daniel Jiang <jiangd@vmware.com>
This commit is contained in:
Daniel Jiang
parent
056ccbac41
commit
1832699e93
14
Makefile
14
Makefile
@ -102,6 +102,7 @@ CLAIRVERSION=v2.0.7
|
||||
CLAIRDBVERSION=$(VERSIONTAG)
|
||||
MIGRATORVERSION=$(VERSIONTAG)
|
||||
REDISVERSION=$(VERSIONTAG)
|
||||
NOTARYMIGRATEVERSION=v3.5.4
|
||||
|
||||
# version of chartmuseum
|
||||
CHARTMUSEUMVERSION=v0.8.1
|
||||
@ -132,11 +133,13 @@ GOBUILDPATH_ADMINSERVER=$(GOBUILDPATH)/src/adminserver
|
||||
GOBUILDPATH_CORE=$(GOBUILDPATH)/src/core
|
||||
GOBUILDPATH_JOBSERVICE=$(GOBUILDPATH)/src/jobservice
|
||||
GOBUILDPATH_REGISTRYCTL=$(GOBUILDPATH)/src/registryctl
|
||||
GOBUILDPATH_MIGRATEPATCH=$(GOBUILDPATH)/src/cmd/migrate-patch
|
||||
GOBUILDMAKEPATH=$(GOBUILDPATH)/make
|
||||
GOBUILDMAKEPATH_ADMINSERVER=$(GOBUILDMAKEPATH)/photon/adminserver
|
||||
GOBUILDMAKEPATH_CORE=$(GOBUILDMAKEPATH)/photon/core
|
||||
GOBUILDMAKEPATH_JOBSERVICE=$(GOBUILDMAKEPATH)/photon/jobservice
|
||||
GOBUILDMAKEPATH_REGISTRYCTL=$(GOBUILDMAKEPATH)/photon/registryctl
|
||||
GOBUILDMAKEPATH_NOTARY=$(GOBUILDMAKEPATH)/photon/notary
|
||||
|
||||
# binary
|
||||
ADMINSERVERBINARYPATH=$(MAKEDEVPATH)/adminserver
|
||||
@ -147,6 +150,7 @@ JOBSERVICEBINARYPATH=$(MAKEDEVPATH)/jobservice
|
||||
JOBSERVICEBINARYNAME=harbor_jobservice
|
||||
REGISTRYCTLBINARYPATH=$(MAKEDEVPATH)/registryctl
|
||||
REGISTRYCTLBINARYNAME=harbor_registryctl
|
||||
MIGRATEPATCHBINARYNAME=migrate-patch
|
||||
|
||||
# configfile
|
||||
CONFIGPATH=$(MAKEPATH)
|
||||
@ -288,7 +292,12 @@ compile_registryctl:
|
||||
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_REGISTRYCTL) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -o $(GOBUILDMAKEPATH_REGISTRYCTL)/$(REGISTRYCTLBINARYNAME)
|
||||
@echo "Done."
|
||||
|
||||
compile:check_environment compile_adminserver compile_core compile_jobservice compile_registryctl
|
||||
compile_notary_migrate_patch:
|
||||
@echo "compiling binary for migrate patch (golang image)..."
|
||||
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_MIGRATEPATCH) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -o $(GOBUILDMAKEPATH_NOTARY)/$(MIGRATEPATCHBINARYNAME)
|
||||
@echo "Done."
|
||||
|
||||
compile:check_environment compile_adminserver compile_core compile_jobservice compile_registryctl compile_notary_migrate_patch
|
||||
|
||||
prepare:
|
||||
@echo "preparing..."
|
||||
@ -296,7 +305,7 @@ prepare:
|
||||
|
||||
build:
|
||||
make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) \
|
||||
-e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) \
|
||||
-e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
|
||||
-e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \
|
||||
-e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \
|
||||
-e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER)
|
||||
@ -320,7 +329,6 @@ modify_composefile_clair:
|
||||
@cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRTPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
|
||||
@$(SEDCMD) -i -e 's/__postgresql_version__/$(CLAIRDBVERSION)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
|
||||
@$(SEDCMD) -i -e 's/__clair_version__/$(CLAIRVERSION)-$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
|
||||
|
||||
modify_composefile_chartmuseum:
|
||||
@echo "preparing docker-compose chartmuseum file..."
|
||||
@cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECHARTMUSEUMTPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECHARTMUSEUMFILENAME)
|
||||
|
||||
@ -167,13 +167,11 @@ _build_notary:
|
||||
@if [ "$(NOTARYFLAG)" = "true" ] ; then \
|
||||
if [ "$(BUILDBIN)" != "true" ] ; then \
|
||||
rm -rf $(DOCKERFILEPATH_NOTARY)/binary && mkdir -p $(DOCKERFILEPATH_NOTARY)/binary && \
|
||||
$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/release-$(NOTARYVERSION)/notary-signer, $(DOCKERFILEPATH_NOTARY)/binary/notary-signer) && \
|
||||
$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/release-$(NOTARYVERSION)/notary-server, $(DOCKERFILEPATH_NOTARY)/binary/notary-server) ; \
|
||||
$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/release-$(NOTARYVERSION)/binary-bundle.tgz, $(DOCKERFILEPATH_NOTARY)/binary-bundle.tgz); \
|
||||
cd $(DOCKERFILEPATH_NOTARY) && tar -zvxf binary-bundle.tgz && cd - ; \
|
||||
else \
|
||||
cd $(DOCKERFILEPATH_NOTARY) && $(DOCKERFILEPATH_NOTARY)/builder $(NOTARYVERSION) && cd - ; \
|
||||
cd $(DOCKERFILEPATH_NOTARY) && $(DOCKERFILEPATH_NOTARY)/builder $(NOTARYVERSION) $(NOTARYMIGRATEVERSION) && cd - ; \
|
||||
fi ; \
|
||||
$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/release-$(NOTARYVERSION)/notary-migrate-postgresql.tgz, $(DOCKERFILEPATH_NOTARY)/binary/notary-migrate.tgz); \
|
||||
cd $(DOCKERFILEPATH_NOTARY)/binary && tar -zvxf notary-migrate.tgz && cd - ; \
|
||||
echo "building notary container for photon..."; \
|
||||
chmod 655 $(DOCKERFILEPATH_NOTARY)/binary/notary-signer && $(DOCKERBUILD) -f $(DOCKERFILEPATH_NOTARY)/$(DOCKERFILENAME_NOTARYSIGNER) -t $(DOCKERIMAGENAME_NOTARYSIGNER):$(NOTARYVERSION)-$(VERSIONTAG) . ; \
|
||||
chmod 655 $(DOCKERFILEPATH_NOTARY)/binary/notary-server && $(DOCKERBUILD) -f $(DOCKERFILEPATH_NOTARY)/$(DOCKERFILENAME_NOTARYSERVER) -t $(DOCKERIMAGENAME_NOTARYSERVER):$(NOTARYVERSION)-$(VERSIONTAG) . ; \
|
||||
|
||||
@ -1,8 +1,13 @@
|
||||
FROM golang:1.11.2
|
||||
|
||||
ARG NOTARY_VERSION
|
||||
ARG MIGRATE_VERSION
|
||||
RUN test -n "$NOTARY_VERSION"
|
||||
RUN test -n "$MIGRATE_VERSION"
|
||||
ENV NOTARYPKG github.com/theupdateframework/notary
|
||||
ENV MIGRATEPKG github.com/golang-migrate/migrate
|
||||
|
||||
COPY . /go/src/${NOTARYPKG}
|
||||
RUN git clone -b $NOTARY_VERSION https://github.com/theupdateframework/notary.git /go/src/${NOTARYPKG}
|
||||
WORKDIR /go/src/${NOTARYPKG}
|
||||
|
||||
RUN go install -tags pkcs11 \
|
||||
@ -10,3 +15,16 @@ RUN go install -tags pkcs11 \
|
||||
|
||||
RUN go install -tags pkcs11 \
|
||||
-ldflags "-w -X ${NOTARYPKG}/version.GitCommit=`git rev-parse --short HEAD` -X ${NOTARYPKG}/version.NotaryVersion=`cat NOTARY_VERSION`" ${NOTARYPKG}/cmd/notary-signer
|
||||
RUN cp -r /go/src/${NOTARYPKG}/migrations/ /
|
||||
|
||||
RUN git clone -b $MIGRATE_VERSION https://github.com/golang-migrate/migrate /go/src/${MIGRATEPKG}
|
||||
WORKDIR /go/src/${MIGRATEPKG}
|
||||
|
||||
RUN curl -fsSL -o /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 && chmod +x /usr/local/bin/dep
|
||||
RUN dep ensure -vendor-only
|
||||
|
||||
ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse"
|
||||
ENV SOURCES="file go_bindata github aws_s3 google_cloud_storage"
|
||||
|
||||
RUN go install -tags "$DATABASES $SOURCES" -ldflags="-X main.Version=${MIGRATE_VERSION}" ${MIGRATEPKG}/cli && mv /go/bin/cli /go/bin/migrate
|
||||
|
||||
|
||||
@ -2,43 +2,28 @@
|
||||
|
||||
set +e
|
||||
|
||||
if [ -z $1 ]; then
|
||||
error "Please set the 'version' variable"
|
||||
if [ -z $2 ]; then
|
||||
error "Please set the notary and migrate version"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
VERSION="$1"
|
||||
|
||||
echo "Building notary and golang-migrate from source, notary version: $1, golang-migrate version: $2"
|
||||
set -e
|
||||
|
||||
# the temp folder to store binary file...
|
||||
mkdir -p binary
|
||||
rm -rf binary/notary-server || true
|
||||
rm -rf binary/notary-signer || true
|
||||
rm -rf binary/* || true
|
||||
|
||||
cd `dirname $0`
|
||||
cur=$PWD
|
||||
docker build --build-arg NOTARY_VERSION=$1 --build-arg MIGRATE_VERSION=$2 -f ./binary.Dockerfile -t notary-binary .
|
||||
|
||||
# the temp folder to store notary source code...
|
||||
TEMP=`mktemp -d /$TMPDIR/notary.XXXXXX`
|
||||
git clone -b $VERSION https://github.com/theupdateframework/notary.git $TEMP
|
||||
|
||||
echo 'build the notary binary bases on the golang:1.11.2...'
|
||||
cp binary.Dockerfile $TEMP
|
||||
cd $TEMP
|
||||
docker build -f binary.Dockerfile -t notary-golang $TEMP
|
||||
cp -r $TEMP/migrations binary
|
||||
|
||||
echo 'copy the notary binary to local...'
|
||||
ID=$(docker create notary-golang)
|
||||
echo $ID
|
||||
cd $cur
|
||||
docker cp $ID:/go/bin/notary-server binary
|
||||
docker cp $ID:/go/bin/notary-signer binary
|
||||
echo 'copy the binary files to local...'
|
||||
ID=$(docker create notary-binary)
|
||||
docker cp $ID:/go/bin/notary-server binary/
|
||||
docker cp $ID:/go/bin/notary-signer binary/
|
||||
docker cp $ID:/go/bin/migrate binary/
|
||||
docker cp $ID:/migrations binary/
|
||||
|
||||
docker rm -f $ID
|
||||
docker rmi -f notary-golang
|
||||
|
||||
rm -rf $TEMP
|
||||
|
||||
docker rmi -f notary-binary
|
||||
|
||||
|
||||
@ -1,2 +1,2 @@
|
||||
#!/bin/sh
|
||||
sudo -E -u \#10000 sh -c "/usr/bin/env /migrations/migrate.sh && /bin/notary-server -config=/etc/notary/server-config.postgres.json -logf=logfmt"
|
||||
sudo -E -u \#10000 sh -c "migrate-patch -database='${DB_URL}' && /migrations/migrate.sh && /bin/notary-server -config=/etc/notary/server-config.postgres.json -logf=logfmt"
|
||||
|
||||
@ -5,10 +5,11 @@ RUN tdnf install -y shadow sudo \
|
||||
&& groupadd -r -g 10000 notary \
|
||||
&& useradd --no-log-init -r -g 10000 -u 10000 notary
|
||||
|
||||
COPY ./make/photon/notary/migrate-patch /bin/migrate-patch
|
||||
COPY ./make/photon/notary/binary/notary-server /bin/notary-server
|
||||
COPY ./make/photon/notary/binary/migrate /bin/migrate
|
||||
COPY ./make/photon/notary/binary/migrations/ /migrations/
|
||||
COPY ./make/photon/notary/server-start.sh /bin/server-start.sh
|
||||
RUN chmod u+x /bin/notary-server /migrations/migrate.sh /bin/migrate /bin/server-start.sh
|
||||
RUN chmod +x /bin/notary-server /migrations/migrate.sh /bin/migrate /bin/migrate-patch /bin/server-start.sh
|
||||
ENV SERVICE_NAME=notary_server
|
||||
ENTRYPOINT [ "/bin/server-start.sh" ]
|
||||
|
||||
@ -1,2 +1,2 @@
|
||||
#!/bin/sh
|
||||
sudo -E -u \#10000 sh -c "/usr/bin/env && /migrations/migrate.sh && /bin/notary-signer -config=/etc/notary/signer-config.postgres.json -logf=logfmt"
|
||||
sudo -E -u \#10000 sh -c "migrate-patch -database='${DB_URL}' && /migrations/migrate.sh && /bin/notary-signer -config=/etc/notary/signer-config.postgres.json -logf=logfmt"
|
||||
|
||||
@ -4,11 +4,12 @@ RUN tdnf install -y shadow sudo \
|
||||
&& tdnf clean all \
|
||||
&& groupadd -r -g 10000 notary \
|
||||
&& useradd --no-log-init -r -g 10000 -u 10000 notary
|
||||
COPY ./make/photon/notary/migrate-patch /bin/migrate-patch
|
||||
COPY ./make/photon/notary/binary/notary-signer /bin/notary-signer
|
||||
COPY ./make/photon/notary/binary/migrate /bin/migrate
|
||||
COPY ./make/photon/notary/binary/migrations/ /migrations/
|
||||
COPY ./make/photon/notary/signer-start.sh /bin/signer-start.sh
|
||||
|
||||
RUN chmod u+x /bin/notary-signer /migrations/migrate.sh /bin/migrate /bin/signer-start.sh
|
||||
RUN chmod +x /bin/notary-signer /migrations/migrate.sh /bin/migrate /bin/migrate-patch /bin/signer-start.sh
|
||||
ENV SERVICE_NAME=notary_signer
|
||||
ENTRYPOINT [ "/bin/signer-start.sh" ]
|
||||
|
||||
6
src/cmd/migrate-patch/README.md
Normal file
6
src/cmd/migrate-patch/README.md
Normal file
@ -0,0 +1,6 @@
|
||||
# Migrate Patch
|
||||
This is a simple program to fix the breakage that was introduced by migrate in notary.
|
||||
## Usage
|
||||
```sh
|
||||
patch -database <db_url>
|
||||
```
|
||||
71
src/cmd/migrate-patch/main.go
Normal file
71
src/cmd/migrate-patch/main.go
Normal file
@ -0,0 +1,71 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
_ "github.com/lib/pq"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var dbURL string
|
||||
|
||||
const pgSQLAlterStmt string = `ALTER TABLE schema_migrations ADD COLUMN "dirty" boolean NOT NULL DEFAULT false`
|
||||
const pgSQLCheckColStmt string = `SELECT T1.C1, T2.C2 FROM
|
||||
(SELECT COUNT(*) AS C1 FROM information_schema.tables WHERE table_name='schema_migrations') T1,
|
||||
(SELECT COUNT(*) AS C2 FROM information_schema.columns WHERE table_name='schema_migrations' and column_name='dirty') T2`
|
||||
const pgSQLDelRows string = `DELETE FROM schema_migrations t WHERE t.version < ( SELECT MAX(version) FROM schema_migrations )`
|
||||
|
||||
func init() {
|
||||
urlUsage := `The URL to the target database (driver://url). Currently it only supports postgres`
|
||||
flag.StringVar(&dbURL, "database", "", urlUsage)
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
log.Printf("Updating database.")
|
||||
if !strings.HasPrefix(dbURL, "postgres://") {
|
||||
log.Fatalf("Invalid URL: '%s'\n", dbURL)
|
||||
}
|
||||
db, err := sql.Open("postgres", dbURL)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to connect to Database, error: %v\n", err)
|
||||
}
|
||||
defer db.Close()
|
||||
c := make(chan int, 1)
|
||||
go func() {
|
||||
err := db.Ping()
|
||||
for ; err != nil; err = db.Ping() {
|
||||
log.Printf("Failed to Ping DB, sleep for 1 second.\n")
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
c <- 1
|
||||
}()
|
||||
select {
|
||||
case <-c:
|
||||
case <-time.After(30 * time.Second):
|
||||
log.Fatal("Failed to connect DB after 30 seconds, time out. \n")
|
||||
|
||||
}
|
||||
row := db.QueryRow(pgSQLCheckColStmt)
|
||||
var tblCount, colCount int
|
||||
if err := row.Scan(&tblCount, &colCount); err != nil {
|
||||
log.Fatalf("Failed to check schema_migrations table, error: %v \n", err)
|
||||
}
|
||||
if tblCount == 0 {
|
||||
log.Printf("schema_migrations table does not exist, skip.\n")
|
||||
return
|
||||
}
|
||||
if colCount > 0 {
|
||||
log.Printf("schema_migrations table does not require update, skip.\n")
|
||||
return
|
||||
}
|
||||
if _, err := db.Exec(pgSQLDelRows); err != nil {
|
||||
log.Fatalf("Failed to clean up table, error: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(pgSQLAlterStmt); err != nil {
|
||||
log.Fatalf("Failed to update database, error: %v \n", err)
|
||||
}
|
||||
log.Printf("Done updating database. \n")
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user