Unify images tags and build process

.travis.yml

@@ -70,7 +70,7 @@ install:
 before_script:
   # create tables and load data
   # - mysql < ./make/db/registry.sql -uroot --verbose
-  - sudo sqlite3 /tmp/registry.db < make/common/db/registry_sqlite.sql
+  - sudo sqlite3 /tmp/registry.db < make/photon/db/registry_sqlite.sql
   - sudo chmod 777 /tmp/registry.db
 
 script:
@@ -82,6 +82,12 @@ script:
   - sudo make run_clarity_ut CLARITYIMAGE=vmware/harbor-clarity-ui-builder:1.2.7
   - cat ./src/ui_ng/lib/npm-ut-test-results
   - sudo ./tests/testprepare.sh 
+  - sudo make -f make/photon/Makefile -e MARIADBVERSION=10.2.10 -e VERSIONTAG=dev
+  - sudo make -f make/photon/Makefile _build_registry -e REGISTRYVERSION=v2.6.2 -e VERSIONTAG=dev
+  - sudo sed -i 's/__reg_version__/v2.6.2-dev/g' ./make/docker-compose.test.yml
+  - sudo sed -i 's/__version__/dev/g' ./make/docker-compose.test.yml
+  - sudo mkdir -p ./make/common/config/registry/
+  - sudo mv ./tests/reg_config.yml ./make/common/config/registry/config.yml
   - sudo docker-compose -f ./make/docker-compose.test.yml up -d
   - go list ./... | grep -v -E 'vendor|tests' | xargs -L1 fgt golint
   - go list ./... | grep -v -E 'vendor|tests' | xargs -L1 go vet

Makefile

@@ -12,12 +12,8 @@
 #							golang:1.7.3
 # compile_adminserver, compile_ui, compile_jobservice: compile specific binary
 #
-# build: 		build Harbor docker images (default: build_photon)
+# build:	build Harbor docker images from photon baseimage
-#			for example: make build -e BASEIMAGE=photon
-# build_photon:	build Harbor docker images from photon baseimage
 #
-# build_postgresql: build postgresql images basaed on photon os
-#       make build -e BASEIMAGE=postgresql
 # install:		include compile binarys, build images, prepare specific \
 #				version composefile and startup Harbor instance
 #
@@ -75,26 +71,40 @@ UIPATH=$(BUILDPATH)/src/ui
 UINGPATH=$(BUILDPATH)/src/ui_ng
 GOBASEPATH=/go/src/github.com/vmware
 CHECKENVCMD=checkenv.sh
-BASEIMAGE=photon
+
-COMPILETAG=compile_normal
+# parameters
 REGISTRYSERVER=
 REGISTRYPROJECTNAME=vmware
 DEVFLAG=true
 NOTARYFLAG=false
-REGISTRYVERSION=2.6.2-photon
+CLAIRFLAG=false
-NGINXVERSION=1.11.13
-PHOTONVERSION=1.0
-NOTARYVERSION=server-0.5.1
-NOTARYSIGNERVERSION=signer-0.5.1
-MARIADBVERSION=10.2.10
 HTTPPROXY=
 REBUILDCLARITYFLAG=false
 NEWCLARITYVERSION=
+BUILDBIN=false
+MIGRATORFLAG=false
 
-#clair parameters
+# version prepare
-CLAIRVERSION=v2.0.1-photon
+VERSIONFILEPATH=$(CURDIR)
-CLAIRFLAG=false
+VERSIONFILENAME=VERSION
-CLAIRDBVERSION=9.6.5-photon
+GITCMD=$(shell which git)
+GITTAG=$(GITCMD) describe --tags
+GITTAGVERSION=$(shell git describe --tags || echo UNKNOWN)
+ifeq ($(DEVFLAG), true)
+	VERSIONTAG=dev
+else
+	VERSIONTAG=$(GITTAGVERSION)
+endif
+
+#versions
+REGISTRYVERSION=v2.6.2
+NGINXVERSION=$(VERSIONTAG)
+PHOTONVERSION=1.0
+NOTARYVERSION=v0.5.1
+MARIADBVERSION=$(VERSIONTAG)
+CLAIRVERSION=v2.0.1
+CLAIRDBVERSION=$(VERSIONTAG)
+MIGRATORVERSION=1.3
 
 #clarity parameters
 CLARITYIMAGE=vmware/harbor-clarity-ui-builder[:tag]
@@ -134,13 +144,10 @@ GOBUILDMAKEPATH_JOBSERVICE=$(GOBUILDMAKEPATH)/dev/jobservice
 GOLANGDOCKERFILENAME=Dockerfile.golang
 
 # binary
-ADMINSERVERSOURCECODE=$(SRCPATH)/adminserver
 ADMINSERVERBINARYPATH=$(MAKEDEVPATH)/adminserver
 ADMINSERVERBINARYNAME=harbor_adminserver
-UISOURCECODE=$(SRCPATH)/ui
 UIBINARYPATH=$(MAKEDEVPATH)/ui
 UIBINARYNAME=harbor_ui
-JOBSERVICESOURCECODE=$(SRCPATH)/jobservice
 JOBSERVICEBINARYPATH=$(MAKEDEVPATH)/jobservice
 JOBSERVICEBINARYNAME=harbor_jobservice
 
@@ -164,14 +171,8 @@ MAKEFILEPATH_PHOTON=$(MAKEPATH)/photon
 
 # common dockerfile
 DOCKERFILEPATH_COMMON=$(MAKEPATH)/common
-DOCKERFILEPATH_DB=$(DOCKERFILEPATH_COMMON)/db
-DOCKERFILENAME_DB=Dockerfile
 DOCKERFILE_CLARITY=$(MAKEPATH)/dev/nodeclarity/Dockerfile
 
-DOCKERFILEPATH_POSTGRESQL=$(DOCKERFILEPATH_COMMON)/postgresql
-DOCKERFILENAME_POSTGRESQL=Dockerfile
-
-
 # docker image name
 DOCKERIMAGENAME_ADMINSERVER=vmware/harbor-adminserver
 DOCKERIMAGENAME_UI=vmware/harbor-ui
@@ -179,26 +180,16 @@ DOCKERIMAGENAME_JOBSERVICE=vmware/harbor-jobservice
 DOCKERIMAGENAME_LOG=vmware/harbor-log
 DOCKERIMAGENAME_DB=vmware/harbor-db
 DOCKERIMAGENAME_CLARITY=vmware/harbor-clarity-ui-builder
-DOCKERIMAGENAME_POSTGRESQL=vmware/postgresql
+
 # docker-compose files
 DOCKERCOMPOSEFILEPATH=$(MAKEPATH)
 DOCKERCOMPOSETPLFILENAME=docker-compose.tpl
 DOCKERCOMPOSEFILENAME=docker-compose.yml
+DOCKERCOMPOSENOTARYTPLFILENAME=docker-compose.notary.tpl
 DOCKERCOMPOSENOTARYFILENAME=docker-compose.notary.yml
+DOCKERCOMPOSECLAIRTPLFILENAME=docker-compose.clair.tpl
 DOCKERCOMPOSECLAIRFILENAME=docker-compose.clair.yml
 
-# version prepare
-VERSIONFILEPATH=$(CURDIR)
-VERSIONFILENAME=VERSION
-GITCMD=$(shell which git)
-GITTAG=$(GITCMD) describe --tags
-GITTAGVERSION=$(shell git describe --tags || echo UNKNOWN)
-ifeq ($(DEVFLAG), true)
-	VERSIONTAG=dev
-else
-	VERSIONTAG=$(GITTAGVERSION)
-endif
-
 SEDCMD=$(shell which sed)
 
 # package
@@ -213,42 +204,36 @@ PUSHSCRIPTNAME=pushimage.sh
 REGISTRYUSER=user
 REGISTRYPASSWORD=default
 
-# migrator
-MIGRATORVERSION=1.3
-MIGRATORFLAG=false
-
 # cmds
 DOCKERSAVE_PARA=$(DOCKERIMAGENAME_ADMINSERVER):$(VERSIONTAG) \
 		$(DOCKERIMAGENAME_UI):$(VERSIONTAG) \
 		$(DOCKERIMAGENAME_LOG):$(VERSIONTAG) \
 		$(DOCKERIMAGENAME_DB):$(VERSIONTAG) \
 		$(DOCKERIMAGENAME_JOBSERVICE):$(VERSIONTAG) \
-		vmware/nginx-photon:$(NGINXVERSION) vmware/registry:$(REGISTRYVERSION) \
+		vmware/nginx-photon:$(NGINXVERSION)-$(VERSIONTAG) vmware/registry-photon:$(REGISTRYVERSION)-$(VERSIONTAG) \
 		vmware/photon:$(PHOTONVERSION)
 PACKAGE_OFFLINE_PARA=-zcvf harbor-offline-installer-$(GITTAGVERSION).tgz \
 		          $(HARBORPKG)/common/templates $(HARBORPKG)/$(DOCKERIMGFILE).$(VERSIONTAG).tar.gz \
 				  $(HARBORPKG)/prepare $(HARBORPKG)/NOTICE \
-				  $(HARBORPKG)/upgrade $(HARBORPKG)/harbor_1_1_0_template \
 				  $(HARBORPKG)/LICENSE $(HARBORPKG)/install.sh \
 				  $(HARBORPKG)/harbor.cfg $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) \
 				  $(HARBORPKG)/ha
 PACKAGE_ONLINE_PARA=-zcvf harbor-online-installer-$(GITTAGVERSION).tgz \
 		          $(HARBORPKG)/common/templates $(HARBORPKG)/prepare \
 				  $(HARBORPKG)/LICENSE $(HARBORPKG)/NOTICE \
-				  $(HARBORPKG)/upgrade $(HARBORPKG)/harbor_1_1_0_template \
 				  $(HARBORPKG)/install.sh $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) \
 				  $(HARBORPKG)/harbor.cfg $(HARBORPKG)/ha
 DOCKERCOMPOSE_LIST=-f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
 
 ifeq ($(NOTARYFLAG), true)
-	DOCKERSAVE_PARA+= vmware/notary-photon:$(NOTARYVERSION) vmware/notary-photon:$(NOTARYSIGNERVERSION) \
+	DOCKERSAVE_PARA+= vmware/notary-server-photon:$(NOTARYVERSION)-$(VERSIONTAG) vmware/notary-signer-photon:$(NOTARYVERSION)-$(VERSIONTAG) \
-				vmware/mariadb-photon:$(MARIADBVERSION)
+				vmware/mariadb-photon:$(MARIADBVERSION)-$(VERSIONTAG)
 	PACKAGE_OFFLINE_PARA+= $(HARBORPKG)/$(DOCKERCOMPOSENOTARYFILENAME)
 	PACKAGE_ONLINE_PARA+= $(HARBORPKG)/$(DOCKERCOMPOSENOTARYFILENAME)
 	DOCKERCOMPOSE_LIST+= -f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSENOTARYFILENAME)
 endif
 ifeq ($(CLAIRFLAG), true)
-	DOCKERSAVE_PARA+= vmware/clair:$(CLAIRVERSION) vmware/postgresql:$(CLAIRDBVERSION)
+	DOCKERSAVE_PARA+= vmware/clair-photon:$(CLAIRVERSION)-$(VERSIONTAG) vmware/postgresql-photon:$(CLAIRDBVERSION)-$(VERSIONTAG)
 	PACKAGE_OFFLINE_PARA+= $(HARBORPKG)/$(DOCKERCOMPOSECLAIRFILENAME)
 	PACKAGE_ONLINE_PARA+= $(HARBORPKG)/$(DOCKERCOMPOSECLAIRFILENAME)
 	DOCKERCOMPOSE_LIST+= -f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
@@ -263,21 +248,6 @@ version:
 check_environment:
 	@$(MAKEPATH)/$(CHECKENVCMD)
 
-compile_adminserver:
-	@echo "compiling binary for adminserver..."
-	@$(GOBUILD) -o $(ADMINSERVERBINARYPATH)/$(ADMINSERVERBINARYNAME) $(ADMINSERVERSOURCECODE)
-	@echo "Done."
-
-compile_ui:
-	@echo "compiling binary for ui..."
-	@$(GOBUILD) -o $(UIBINARYPATH)/$(UIBINARYNAME) $(UISOURCECODE)
-	@echo "Done."
-
-compile_jobservice:
-	@echo "compiling binary for jobservice..."
-	@$(GOBUILD) -o $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) $(JOBSERVICESOURCECODE)
-	@echo "Done."
-
 compile_clarity:
 	@echo "compiling binary for clarity ui..."
 	@if [ "$(HTTPPROXY)" != "" ] ; then \
@@ -287,8 +257,6 @@ compile_clarity:
 	fi
 	@echo "Done."
 
-compile_normal: compile_clarity compile_adminserver compile_ui compile_jobservice
-
 compile_golangimage: compile_clarity
 	@echo "compiling binary for adminserver (golang image)..."
 	@echo $(GOBASEPATH)
@@ -306,31 +274,38 @@ compile_golangimage: compile_clarity
 	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_JOBSERVICE) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -v -o $(GOBUILDMAKEPATH_JOBSERVICE)/$(JOBSERVICEBINARYNAME)
 	@echo "Done."
 
-compile:check_environment $(COMPILETAG)
+compile:check_environment compile_golangimage
 	
 prepare:
 	@echo "preparing..."
 	@$(MAKEPATH)/$(PREPARECMD) $(PREPARECMD_PARA)
 
-build_common: version
+build:
-	@echo "buildging db container for photon..."
+	make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) -e MARIADBVERSION=$(MARIADBVERSION) \
-	@cd $(DOCKERFILEPATH_DB) && $(DOCKERBUILD) --pull -f $(DOCKERFILENAME_DB) -t $(DOCKERIMAGENAME_DB):$(VERSIONTAG) .
+	 -e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) \
-	@echo "Done."
+	 -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \
+	 -e BUILDBIN=$(BUILDBIN)
 
-build_photon: build_common
+modify_composefile: modify_composefile_notary modify_composefile_clair
-	make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG)
-build_postgresql:
-	@echo "buildging postgresql container for photon..."
-	@cd $(DOCKERFILEPATH_POSTGRESQL) && $(DOCKERBUILD) -f $(DOCKERFILENAME_POSTGRESQL) -t $(DOCKERIMAGENAME_POSTGRESQL):$(CLAIRDBVERSION) .
-	@echo "Done."
-build: build_$(BASEIMAGE)
-
-modify_composefile:
 	@echo "preparing docker-compose file..."
 	@cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSETPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
 	@cp $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSETPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSEFILENAME)
-	@$(SEDCMD) -i 's/__version__/$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
 	@$(SEDCMD) -i 's/__version__/$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSEFILENAME)
+	@$(SEDCMD) -i 's/__reg_version__/$(REGISTRYVERSION)-$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
+	@$(SEDCMD) -i 's/__version__/$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
+	@$(SEDCMD) -i 's/__nginx_version__/$(NGINXVERSION)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
+
+modify_composefile_notary:
+	@echo "preparing docker-compose notary file..."
+	@cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSENOTARYTPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSENOTARYFILENAME)
+	@$(SEDCMD) -i 's/__notary_version__/$(NOTARYVERSION)-$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSENOTARYFILENAME)
+	@$(SEDCMD) -i 's/__mariadb_version__/$(MARIADBVERSION)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSENOTARYFILENAME)
+
+modify_composefile_clair:
+	@echo "preparing docker-compose clair file..."
+	@cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRTPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
+	@$(SEDCMD) -i 's/__postgresql_version__/$(CLAIRDBVERSION)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
+	@$(SEDCMD) -i 's/__clair_version__/$(CLAIRVERSION)-$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
 
 modify_sourcefiles:
 	@echo "change mode of source files."
@@ -340,7 +315,7 @@ modify_sourcefiles:
 	@chmod 600 $(MAKEPATH)/common/templates/ui/private_key.pem
 	@chmod 600 $(MAKEPATH)/common/templates/registry/root.crt
 
-install: compile build modify_sourcefiles prepare modify_composefile start
+install: compile version build modify_sourcefiles prepare modify_composefile start
 
 package_online: modify_composefile
 	@echo "packing online package ..."
@@ -353,41 +328,22 @@ package_online: modify_composefile
 	fi
 	@cp LICENSE $(HARBORPKG)/LICENSE
 	@cp NOTICE $(HARBORPKG)/NOTICE
-	@cp tools/migration/migration_cfg/upgrade $(HARBORPKG)/upgrade
+	
-	@cp tools/migration/migration_cfg/harbor_1_1_0_template $(HARBORPKG)/harbor_1_1_0_template
-
 	@$(TARCMD) $(PACKAGE_ONLINE_PARA)
 	@rm -rf $(HARBORPKG)
 	@echo "Done."
 	
-package_offline: compile build modify_sourcefiles modify_composefile
+package_offline: compile version build modify_sourcefiles modify_composefile
 	@echo "packing offline package ..."
 	@cp -r make $(HARBORPKG)
 
 	@cp LICENSE $(HARBORPKG)/LICENSE
 	@cp NOTICE $(HARBORPKG)/NOTICE
-	@cp tools/migration/migration_cfg/upgrade $(HARBORPKG)/upgrade
-	@cp tools/migration/migration_cfg/harbor_1_1_0_template $(HARBORPKG)/harbor_1_1_0_template
 	@cp $(HARBORPKG)/common/db/registry.sql $(HARBORPKG)/ha/
-
-	@echo "pulling nginx and registry..."
-	@$(DOCKERPULL) vmware/registry:$(REGISTRYVERSION)
-	@$(DOCKERPULL) vmware/nginx-photon:$(NGINXVERSION)
-	@if [ "$(NOTARYFLAG)" = "true" ] ; then \
-		echo "pulling notary and harbor-notary-db..."; \
-		$(DOCKERPULL) vmware/notary-photon:$(NOTARYVERSION); \
-		$(DOCKERPULL) vmware/notary-photon:$(NOTARYSIGNERVERSION); \
-		$(DOCKERPULL) vmware/mariadb-photon:$(MARIADBVERSION); \
-	fi
-	@if [ "$(CLAIRFLAG)" = "true" ] ; then \
-		echo "pulling claiy and postgres..."; \
-		$(DOCKERPULL) vmware/clair:$(CLAIRVERSION); \
-		$(DOCKERPULL) vmware/postgresql:$(CLAIRDBVERSION); \
-	fi
 	@if [ "$(MIGRATORFLAG)" = "true" ] ; then \
 		echo "pulling DB migrator..."; \
 		$(DOCKERPULL) vmware/harbor-db-migrator:$(MIGRATORVERSION); \
-	fi	
+	fi
 
 	@echo "saving harbor docker image"
 	@$(DOCKERSAVE) $(DOCKERSAVE_PARA) > $(HARBORPKG)/$(DOCKERIMGFILE).$(VERSIONTAG).tar
@@ -474,8 +430,6 @@ cleanimage:
 	- $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_DB):$(VERSIONTAG)
 	- $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_JOBSERVICE):$(VERSIONTAG)
 	- $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_LOG):$(VERSIONTAG)
-#	- $(DOCKERRMIMAGE) -f registry:$(REGISTRYVERSION)
-#	- $(DOCKERRMIMAGE) -f nginx:1.11.5
 
 cleandockercomposefile:
 	@echo "cleaning $(DOCKERCOMPOSEFILEPATH)/docker-compose.yml"

make/docker-compose.clair.tpl

@@ -17,7 +17,7 @@ services:
         aliases:
           - postgres
     container_name: clair-db
-    image: vmware/postgresql:9.6.5-photon
+    image: vmware/postgresql-photon:__postgresql_version__
     restart: always
     depends_on:
       - log
@@ -35,7 +35,7 @@ services:
     networks:
       - harbor-clair
     container_name: clair
-    image: vmware/clair:v2.0.1-photon
+    image: vmware/clair-photon:__clair_version__
     restart: always
     cpu_quota: 150000
     depends_on:

make/docker-compose.notary.tpl

@@ -7,7 +7,7 @@ services:
     networks:
       - harbor-notary
   notary-server:
-    image: vmware/notary-photon:server-0.5.1
+    image: vmware/notary-server-photon:__notary_version__
     container_name: notary-server
     restart: always
     networks:
@@ -25,7 +25,7 @@ services:
         syslog-address: "tcp://127.0.0.1:1514"
         tag: "notary-server"
   notary-signer:
-    image: vmware/notary-photon:signer-0.5.1
+    image: vmware/notary-signer-photon:__notary_version__
     container_name: notary-signer
     restart: always
     networks:
@@ -45,7 +45,7 @@ services:
         syslog-address: "tcp://127.0.0.1:1514"
         tag: "notary-signer"
   notary-db:
-    image: vmware/mariadb-photon:10.2.10
+    image: vmware/mariadb-photon:__mariadb_version__
     container_name: notary-db
     restart: always
     networks:

make/docker-compose.tpl

@@ -12,7 +12,7 @@ services:
     networks:
       - harbor
   registry:
-    image: vmware/registry:2.6.2-photon
+    image: vmware/registry-photon:__reg_version__
     container_name: registry
     restart: always
     volumes:
@@ -112,7 +112,7 @@ services:
         syslog-address: "tcp://127.0.0.1:1514"
         tag: "jobservice"
   proxy:
-    image: vmware/nginx-photon:1.11.13
+    image: vmware/nginx-photon:__nginx_version__
     container_name: nginx
     restart: always
     volumes:

make/photon/Makefile

@@ -11,9 +11,8 @@ BUILDPATH=$(CURDIR)
 MAKEPATH=$(BUILDPATH)/make
 MAKEDEVPATH=$(MAKEPATH)/dev
 SRCPATH=./src
-TOOLSPATH=$(BUILDPATH)/tools
+SEDCMD=$(shell which sed)
-CHECKENVCMD=checkenv.sh
+WGET=$(shell which wget)
-DEVFLAG=true
 
 # docker parameters
 DOCKERCMD=$(shell which docker)
@@ -34,50 +33,146 @@ JOBSERVICEBINARYNAME=harbor_jobservice
 
 # photon dockerfile
 DOCKERFILEPATH=$(MAKEPATH)/photon
+
 DOCKERFILEPATH_ADMINSERVER=$(DOCKERFILEPATH)/adminserver
 DOCKERFILENAME_ADMINSERVER=Dockerfile
 DOCKERIMAGENAME_ADMINSERVER=vmware/harbor-adminserver
+
 DOCKERFILEPATH_UI=$(DOCKERFILEPATH)/ui
 DOCKERFILENAME_UI=Dockerfile
 DOCKERIMAGENAME_UI=vmware/harbor-ui
+
 DOCKERFILEPATH_JOBSERVICE=$(DOCKERFILEPATH)/jobservice
 DOCKERFILENAME_JOBSERVICE=Dockerfile
 DOCKERIMAGENAME_JOBSERVICE=vmware/harbor-jobservice
+
 DOCKERFILEPATH_LOG=$(DOCKERFILEPATH)/log
 DOCKERFILENAME_LOG=Dockerfile
 DOCKERIMAGENAME_LOG=vmware/harbor-log
 
-# version prepare
+DOCKERFILEPATH_DB=$(DOCKERFILEPATH)/db
-VERSIONFILEPATH=$(SRCPATH)/views/sections
+DOCKERFILENAME_DB=Dockerfile
-VERSIONFILENAME=header-content.htm
+DOCKERIMAGENAME_DB=vmware/harbor-db
-GITCMD=$(shell which git)
-GITTAG=$(GITCMD) describe --tags
-ifeq ($(DEVFLAG), true)        
-	VERSIONTAG=dev
-else        
-	VERSIONTAG=$(shell $(GITTAG))
-endif
 
-check_environment:
+DOCKERFILEPATH_POSTGRESQL=$(DOCKERFILEPATH)/postgresql
-	@$(MAKEPATH)/$(CHECKENVCMD)
+DOCKERFILENAME_POSTGRESQL=Dockerfile
+DOCKERIMAGENAME_POSTGRESQL=vmware/postgresql-photon
 
-build: 
+DOCKERFILEPATH_CLAIR=$(DOCKERFILEPATH)/clair
+DOCKERFILENAME_CLAIR=Dockerfile
+DOCKERIMAGENAME_CLAIR=vmware/clair-photon
+
+DOCKERFILEPATH_NGINX=$(DOCKERFILEPATH)/nginx
+DOCKERFILENAME_NGINX=Dockerfile
+DOCKERIMAGENAME_NGINX=vmware/nginx-photon
+
+DOCKERFILEPATH_REG=$(DOCKERFILEPATH)/registry
+DOCKERFILENAME_REG=Dockerfile
+DOCKERIMAGENAME_REG=vmware/registry-photon
+
+DOCKERFILEPATH_MARIADB=$(DOCKERFILEPATH)/mariadb
+DOCKERFILENAME_MARIADB=Dockerfile
+DOCKERIMAGENAME_MARIADB=vmware/mariadb-photon
+
+DOCKERFILEPATH_NOTARY=$(DOCKERFILEPATH)/notary
+DOCKERFILENAME_NOTARYSIGNER=signer.Dockerfile
+DOCKERIMAGENAME_NOTARYSIGNER=vmware/notary-signer-photon
+DOCKERFILENAME_NOTARYSERVER=server.Dockerfile
+DOCKERIMAGENAME_NOTARYSERVER=vmware/notary-server-photon
+
+_build_db: _build_mariadb
+	@echo "modify the db dockerfile..."
+	@$(SEDCMD) -i 's/__version__/$(MARIADBVERSION)/g' $(DOCKERFILEPATH_DB)/$(DOCKERFILENAME_DB)
+	@echo "building db container for photon..."
+	@cd $(DOCKERFILEPATH_DB) && $(DOCKERBUILD) -f $(DOCKERFILEPATH_DB)/$(DOCKERFILENAME_DB) -t $(DOCKERIMAGENAME_DB):$(VERSIONTAG) .
+	@echo "Done."
+	@$(SEDCMD) -i 's/$(MARIADBVERSION)/__version__/g' $(DOCKERFILEPATH_DB)/$(DOCKERFILENAME_DB)
+
+_build_adminiserver:		
 	@echo "building adminserver container for photon..."
 	$(DOCKERBUILD) -f $(DOCKERFILEPATH_ADMINSERVER)/$(DOCKERFILENAME_ADMINSERVER) -t $(DOCKERIMAGENAME_ADMINSERVER):$(VERSIONTAG) .
 	@echo "Done."
 
+_build_ui:	
 	@echo "building ui container for photon..."
 	$(DOCKERBUILD) -f $(DOCKERFILEPATH_UI)/$(DOCKERFILENAME_UI) -t $(DOCKERIMAGENAME_UI):$(VERSIONTAG) .
 	@echo "Done."
 	
+_build_jobservice:		
 	@echo "building jobservice container for photon..."
 	$(DOCKERBUILD) -f $(DOCKERFILEPATH_JOBSERVICE)/$(DOCKERFILENAME_JOBSERVICE) -t $(DOCKERIMAGENAME_JOBSERVICE):$(VERSIONTAG) .
 	@echo "Done."
-	
+
+_build_log:	
 	@echo "building log container for photon..."
 	$(DOCKERBUILD) -f $(DOCKERFILEPATH_LOG)/$(DOCKERFILENAME_LOG) -t $(DOCKERIMAGENAME_LOG):$(VERSIONTAG) $(DOCKERFILEPATH_LOG)
 	@echo "Done."
-	
+
+_build_postgresql:		
+	@echo "building postgresql container for photon..."
+	@cd $(DOCKERFILEPATH_POSTGRESQL) && $(DOCKERBUILD) -f $(DOCKERFILEPATH_POSTGRESQL)/$(DOCKERFILENAME_POSTGRESQL) -t $(DOCKERIMAGENAME_POSTGRESQL):$(CLAIRDBVERSION) .
+	@echo "Done."
+
+_build_clair:
+	@if [ "$(CLAIRFLAG)" = "true" ] ; then \
+		if [ "$(BUILDBIN)" != "true" ] ; then \
+			rm -rf $(DOCKERFILEPATH_CLAIR)/binary && mkdir -p $(DOCKERFILEPATH_CLAIR)/binary && \
+			$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/clair, $(DOCKERFILEPATH_CLAIR)/binary/clair); \
+		else \
+			cd $(DOCKERFILEPATH_CLAIR) && $(DOCKERFILEPATH_CLAIR)/builder $(CLAIRVERSION); \
+		fi ; \
+		echo "building clair container for photon..." ; \
+		cd $(DOCKERFILEPATH_CLAIR) && $(DOCKERBUILD) -f $(DOCKERFILEPATH_CLAIR)/$(DOCKERFILENAME_CLAIR) -t $(DOCKERIMAGENAME_CLAIR):$(CLAIRVERSION)-$(VERSIONTAG) . ; \
+		rm -rf $(DOCKERFILEPATH_CLAIR)/binary; \
+		echo "Done." ; \
+	fi
+		
+_build_nginx:
+	@echo "building nginx container for photon..."
+	@$(DOCKERBUILD) -f $(DOCKERFILEPATH_NGINX)/$(DOCKERFILENAME_NGINX) -t $(DOCKERIMAGENAME_NGINX):$(NGINXVERSION) .
+	@echo "Done."
+
+_build_notary:
+	@if [ "$(NOTARYFLAG)" = "true" ] ; then \
+		if [ "$(BUILDBIN)" != "true" ] ; then \
+			rm -rf $(DOCKERFILEPATH_NOTARY)/binary && mkdir -p $(DOCKERFILEPATH_NOTARY)/binary && \
+			$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/notary-signer, $(DOCKERFILEPATH_NOTARY)/binary/notary-signer) && \
+			$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/notary-server, $(DOCKERFILEPATH_NOTARY)/binary/notary-server) && \
+			$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/notary/notary-migrate.tgz, $(DOCKERFILEPATH_NOTARY)/binary/notary-migrate.tgz); \
+			cd $(DOCKERFILEPATH_NOTARY)/binary && tar -zvxf notary-migrate.tgz; \
+		else \
+			cd $(DOCKERFILEPATH_NOTARY) && $(DOCKERFILEPATH_NOTARY)/builder_public $(NOTARYVERSION); \
+		fi ; \
+		echo "building notary container for photon..."; \
+		cd $(DOCKERFILEPATH_NOTARY) && chmod 655 $(DOCKERFILEPATH_NOTARY)/binary/notary-signer && $(DOCKERBUILD) -f $(DOCKERFILEPATH_NOTARY)/$(DOCKERFILENAME_NOTARYSIGNER) -t $(DOCKERIMAGENAME_NOTARYSIGNER):$(NOTARYVERSION)-$(VERSIONTAG) . ; \
+		cd $(DOCKERFILEPATH_NOTARY) && chmod 655 $(DOCKERFILEPATH_NOTARY)/binary/notary-server && $(DOCKERBUILD) -f $(DOCKERFILEPATH_NOTARY)/$(DOCKERFILENAME_NOTARYSERVER) -t $(DOCKERIMAGENAME_NOTARYSERVER):$(NOTARYVERSION)-$(VERSIONTAG) . ; \
+		rm -rf $(DOCKERFILEPATH_NOTARY)/binary; \
+		echo "Done."; \
+	fi
+		
+_build_registry:
+	@if [ "$(BUILDBIN)" != "true" ] ; then \
+		rm -rf $(DOCKERFILEPATH_REG)/binary && mkdir -p $(DOCKERFILEPATH_REG)/binary && \
+		$(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/registry, $(DOCKERFILEPATH_REG)/binary/registry); \
+	else \
+		cd $(DOCKERFILEPATH_REG) && $(DOCKERFILEPATH_REG)/builder $(REGISTRYVERSION); \
+	fi
+	@echo "building registry container for photon..."
+	@cd $(DOCKERFILEPATH_REG) && chmod 655 $(DOCKERFILEPATH_REG)/binary/registry && $(DOCKERBUILD) -f $(DOCKERFILEPATH_REG)/$(DOCKERFILENAME_REG) -t $(DOCKERIMAGENAME_REG):$(REGISTRYVERSION)-$(VERSIONTAG) .
+	@rm -rf $(DOCKERFILEPATH_REG)/binary
+	@echo "Done."
+			
+_build_mariadb:	
+	@echo "building mariadb container for photon..."
+	@cd $(DOCKERFILEPATH_MARIADB) && $(DOCKERBUILD) -f $(DOCKERFILEPATH_MARIADB)/$(DOCKERFILENAME_MARIADB) -t $(DOCKERIMAGENAME_MARIADB):$(MARIADBVERSION) .
+	@echo "Done."
+
+define _get_binary
+	$(WGET) --timeout 30 --no-check-certificate $1 -O $2
+endef
+
+build: _build_postgresql _build_db _build_adminiserver _build_ui _build_jobservice _build_log _build_nginx _build_registry _build_notary _build_clair
+
 cleanimage:
 	@echo "cleaning image for photon..."
 	- $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_ADMINSERVER):$(VERSIONTAG)

make/photon/clair/Dockerfile

@@ -8,7 +8,7 @@ RUN tdnf distro-sync -y \
     && mkdir /clair2.0.1/ \
     && groupadd -r -g 10000 clair \
     && useradd --no-log-init -m -r -g 10000 -u 10000 clair
-COPY clair /clair2.0.1/
+COPY ./binary/clair /clair2.0.1/
 COPY docker-entrypoint.sh /docker-entrypoint.sh
 COPY dumb-init /dumb-init 
 

make/photon/clair/Dockerfile.binary

@@ -0,0 +1,6 @@
+FROM golang:1.7.3
+
+ADD .   /go/src/github.com/coreos/clair/
+WORKDIR /go/src/github.com/coreos/clair/
+
+RUN go install -v github.com/coreos/clair/cmd/clair

make/photon/clair/builder

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set +e
+
+if [ -z $1 ]; then
+  error "Please set the 'version' variable"
+  exit 1
+fi
+
+VERSION="$1"
+
+set -e
+
+# the temp folder to store binary file...
+mkdir -p binary
+rm -rf binary/clair || true
+
+cd `dirname $0`
+cur=$PWD
+
+# the temp folder to store distribution source code...
+TEMP=`mktemp -d /$TMPDIR/clair.XXXXXX`
+git clone -b $VERSION https://github.com/coreos/clair.git $TEMP
+
+echo 'build the clair binary bases on the golang:1.7.3...'
+cp Dockerfile.binary $TEMP
+docker build -f $TEMP/Dockerfile.binary -t clair-golang $TEMP
+
+echo 'copy the clair binary to local...'
+ID=$(docker create clair-golang)
+docker cp $ID:/go/bin/clair binary
+
+docker rm -f $ID
+docker rmi -f clair-golang
+
+echo "Build clair binary success, then to build photon image..."
+cd $cur
+rm -rf $TEMP

make/photon/db/Dockerfile

@@ -1,4 +1,4 @@
-FROM vmware/mariadb-photon:10.2.10
+FROM vmware/mariadb-photon:__version__
 
 HEALTHCHECK CMD mysqladmin -uroot -p$MYSQL_ROOT_PASSWORD ping
 

make/photon/mariadb/Dockerfile

@@ -22,4 +22,4 @@ VOLUME /var/lib/mysql /docker-entrypoint-initdb.d /docker-entrypoint-updatedb.d
 EXPOSE 3306
 
 ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
-CMD ["mysqld"]
+CMD ["mysqld"]

make/photon/notary/binary.Dockerfile

@@ -1,13 +1,12 @@
 FROM golang:1.7.3
 
-ENV NOTARY_DIR /go/src/github.com/docker/notary
+ENV NOTARYPKG github.com/theupdateframework/notary
-ENV NOTARYPKG github.com/docker/notary
 
 COPY . /go/src/${NOTARYPKG}
 WORKDIR /go/src/${NOTARYPKG}
 
-RUN go build -tags pkcs11 \
+RUN go install -tags pkcs11 \
-    -ldflags "-w -X ${NOTARYPKG}/version.GitCommit=`git rev-parse --short HEAD` -X ${NOTARYPKG}/version.NotaryVersion=`cat NOTARY_VERSION`" $NOTARYPKG/cmd/notary-server 
+    -ldflags "-w -X ${NOTARYPKG}/version.GitCommit=`git rev-parse --short HEAD` -X ${NOTARYPKG}/version.NotaryVersion=`cat NOTARY_VERSION`" ${NOTARYPKG}/cmd/notary-server 
 
-RUN go build -tags pkcs11 \
+RUN go install -tags pkcs11 \
-    -ldflags "-w -X ${NOTARYPKG}/version.GitCommit=`git rev-parse --short HEAD` -X ${NOTARYPKG}/version.NotaryVersion=`cat NOTARY_VERSION`" $NOTARYPKG/cmd/notary-signer
+    -ldflags "-w -X ${NOTARYPKG}/version.GitCommit=`git rev-parse --short HEAD` -X ${NOTARYPKG}/version.NotaryVersion=`cat NOTARY_VERSION`" ${NOTARYPKG}/cmd/notary-signer

make/photon/notary/builder_public

@@ -2,26 +2,12 @@
 
 set +e
 
-echo "Usage: #./builder [notary version] [registry username] [registry password]"
 if [ -z $1 ]; then
   error "Please set the 'version' variable"
   exit 1
 fi
-if [ -z $2 ]; then
-  error "Please set the 'photonversion' variable"
-  exit 1
-fi
-if [ -z $3 ]; then
-  error "Please set the 'username' variable"
-  exit 1
-fi
-
 
 VERSION="$1"
-USERNAME="$2"
-PASSWORD="$3"
-SIGNER_PHOTONIMAGE=vmware/notary-photon:signer-$VERSION
-SERVER_PHOTONIMAGE=vmware/notary-photon:server-$VERSION
 
 set -e
 
@@ -41,19 +27,18 @@ echo 'build the notary binary bases on the golang:1.7.3...'
 cp binary.Dockerfile $TEMP 
 cd $TEMP
 docker build -f binary.Dockerfile -t notary-golang $TEMP
+cp -r $TEMP/migrations binary
 
 echo 'copy the notary binary to local...'
 ID=$(docker create notary-golang)
 echo $ID
 cd $cur
-docker cp $ID:/go/src/github.com/docker/notary/notary-server binary
+docker cp $ID:/go/src/github.com/theupdateframework/notary/notary-server binary
-docker cp $ID:/go/src/github.com/docker/notary/notary-signer binary
+docker cp $ID:/go/src/github.com/theupdateframework/notary/notary-signer binary
 
 docker rm -f $ID
 docker rmi -f notary-golang
 
-docker build -f server.Dockerfile -t $SERVER_PHOTONIMAGE .
+rm -rf $TEMP
-docker build -f signer.Dockerfile -t $SIGNER_PHOTONIMAGE .
+
 
-echo 'Push image to docker hub.'
-../../pushimage.sh $PHOTONIMAGE $USERNAME $PASSWORD

make/photon/notary/server.Dockerfile

@@ -8,9 +8,9 @@ RUN tdnf distro-sync -y \
     && useradd --no-log-init -r -g 10000 -u 10000 notary
 
 COPY ./binary/notary-server /bin/notary-server
-COPY ./migrate /bin/migrate
+COPY ./binary/migrate /bin/migrate
-COPY ./migrations/ /migrations/
+COPY ./binary/migrations/ /migrations/
 COPY ./server-start.sh /bin/server-start.sh
 RUN chmod u+x /bin/notary-server /migrations/migrate.sh /bin/migrate /bin/server-start.sh
 ENV SERVICE_NAME=notary_server
-ENTRYPOINT [ "/bin/server-start.sh" ]
+ENTRYPOINT [ "/bin/server-start.sh" ]

make/photon/notary/signer.Dockerfile

@@ -7,10 +7,10 @@ RUN tdnf distro-sync -y \
     && groupadd -r -g 10000 notary \
     && useradd --no-log-init -r -g 10000 -u 10000 notary
 COPY ./binary/notary-signer /bin/notary-signer
-COPY ./migrate /bin/migrate
+COPY ./binary/migrate /bin/migrate
-COPY ./migrations/ /migrations/
+COPY ./binary/migrations/ /migrations/
 COPY ./signer-start.sh /bin/signer-start.sh
 
 RUN chmod u+x /bin/notary-signer /migrations/migrate.sh /bin/migrate /bin/signer-start.sh
 ENV SERVICE_NAME=notary_signer
-ENTRYPOINT [ "/bin/signer-start.sh" ]
+ENTRYPOINT [ "/bin/signer-start.sh" ]

make/photon/registry/Dockerfile

@@ -13,7 +13,6 @@ COPY entrypoint.sh /
 RUN chmod u+x /entrypoint.sh
 
 RUN mkdir -p /etc/registry
-COPY config.yml /etc/registry/config.yml
 
 COPY binary/registry /usr/bin
 RUN chmod u+x /usr/bin/registry

make/photon/registry/builder

@@ -2,29 +2,12 @@
 
 set +e
 
-echo "Usage: #./builder [registry version] [photon tag] [registry username] [registry password]"
 if [ -z $1 ]; then
   error "Please set the 'version' variable"
   exit 1
 fi
-if [ -z $2 ]; then
-  error "Please set the 'photonversion' variable"
-  exit 1
-fi
-if [ -z $3 ]; then
-  error "Please set the 'username' variable"
-  exit 1
-fi
-if [ -z $4 ]; then
-  error "Please set the 'password' variable"
-  exit 1
-fi
 
 VERSION="$1"
-PHOTONVERSION="$2"
-USERNAME="$3"
-PASSWORD="$4"
-PHOTONIMAGE=vmware/registry:$PHOTONVERSION
 
 set -e
 
@@ -52,13 +35,5 @@ docker rmi -f registry-golang
 
 echo "Build registry binary success, then to build photon image..."
 cd $cur
-echo $PHOTONIMAGE
 cp $TEMP/cmd/registry/config-example.yml config.yml
-docker build -f Dockerfile -t $PHOTONIMAGE .
-
 rm -rf $TEMP
-rm -rf binary
-rm -rf config.yml
-
-echo 'Push image to docker hub.'
-../../pushimage.sh $PHOTONIMAGE $USERNAME $PASSWORD

tests/docker-compose.test.yml

@@ -1,7 +1,7 @@
 version: '2'
 services:
   registry:
-    image: vmware/registry:2.6.1-photon
+    image: vmware/registry-photon:__reg_version__
     restart: always
     volumes:
       - /data/registry:/storage
@@ -11,9 +11,9 @@ services:
     ports:
       - 5000:5000
     command:
-      ["serve", "/etc/docker/registry/config.yml"]
+      ["serve", "/etc/registry/config.yml"]
   mysql:
-    build: ./common/db/
+    image: vmware/harbor-db:__version__
     restart: always
     volumes:
       - /data/database:/var/lib/mysql

tests/reg_config.yml

@@ -0,0 +1,18 @@
+version: 0.1
+log:
+  fields:
+    service: registry
+storage:
+  cache:
+    blobdescriptor: inmemory
+  filesystem:
+    rootdirectory: /var/lib/registry
+http:
+  addr: :5000
+  headers:
+    X-Content-Type-Options: [nosniff]
+health:
+  storagedriver:
+    enabled: true
+    interval: 10s
+    threshold: 3