Merge pull request #10047 from bitsf/makefile_clean

optimize the makefile process

.gitignore

@@ -1,5 +1,6 @@
 !/contrib/helm/harbor
 
+make/harbor.yml
 make/docker-compose.yml
 make/common/config/*
 make/dev/adminserver/harbor_adminserver

.travis.yml

@@ -38,7 +38,7 @@ env:
   - CORE_SECRET: tempString
   - KEY_PATH: "/data/secret/keys/secretkey"
   - REDIS_HOST: localhost
-  - REG_VERSION: v2.7.1
+  - REG_VERSION: v2.7.1-patch-2819-2553
   - UI_BUILDER_VERSION: 1.6.0
   - TOKEN_PRIVATE_KEY_PATH: "/home/travis/gopath/src/github.com/goharbor/harbor/tests/private_key.pem"
 addons: 
@@ -56,16 +56,18 @@ before_install:
 - IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'`
 - sudo sed -i '$a DOCKER_OPTS=\"--insecure-registry '$IP':5000\"' /etc/default/docker
 - export IP=$IP
+- sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
+- sudo update-ca-certificates
 - sudo service docker restart
 install:
-- if [ "$UTTEST" == true ]; then bash ./tests/travis/ut_install.sh ; fi
-- if [ "$APITEST_DB" == true ]; then bash ./tests/travis/api_common_install.sh $IP
+- if [ "$UTTEST" == true ]; then bash ./tests/showtime.sh ./tests/travis/ut_install.sh ; fi
+- if [ "$APITEST_DB" == true ]; then bash ./tests/showtime.sh ./tests/travis/api_common_install.sh $IP
   DB; fi
-- if [ "$APITEST_LDAP" == true ]; then bash ./tests/travis/api_common_install.sh $IP
+- if [ "$APITEST_LDAP" == true ]; then bash ./tests/showtime.sh ./tests/travis/api_common_install.sh $IP
   LDAP; fi
 script:
-- if [ "$UTTEST" == true ]; then bash ./tests/travis/ut_run.sh $IP; fi
-- if [ "$APITEST_DB" == true ]; then bash ./tests/travis/api_run.sh DB $IP; fi
-- if [ "$APITEST_LDAP" == true ]; then bash ./tests/travis/api_run.sh LDAP $IP; fi
-- if [ "$OFFLINE" == true ]; then bash ./tests/travis/distro_installer.sh; fi
-- if [ "$UI_UT" == true ]; then bash ./tests/travis/ui_ut_run.sh ; fi
+- if [ "$UTTEST" == true ]; then bash ./tests/showtime.sh ./tests/travis/ut_run.sh $IP; fi
+- if [ "$APITEST_DB" == true ]; then bash ./tests/showtime.sh ./tests/travis/api_run.sh DB $IP; fi
+- if [ "$APITEST_LDAP" == true ]; then bash ./tests/showtime.sh ./tests/travis/api_run.sh LDAP $IP; fi
+- if [ "$OFFLINE" == true ]; then bash ./tests/showtime.sh ./tests/travis/distro_installer.sh; fi
+- if [ "$UI_UT" == true ]; then bash ./tests/showtime.sh ./tests/travis/ui_ut_run.sh ; fi

Makefile

@@ -64,7 +64,6 @@
 SHELL := /bin/bash
 BUILDPATH=$(CURDIR)
 MAKEPATH=$(BUILDPATH)/make
-MAKEDEVPATH=$(MAKEPATH)/dev
 MAKE_PREPARE_PATH=$(MAKEPATH)/photon/prepare
 SRCPATH=./src
 TOOLSPATH=$(BUILDPATH)/tools
@@ -84,6 +83,7 @@ MIGRATORFLAG=false
 NPM_REGISTRY=https://registry.npmjs.org
 # enable/disable chart repo supporting
 CHARTFLAG=false
+BUILDTARGET=build
 
 # version prepare
 # for docker image tag
@@ -139,7 +139,7 @@ GOTEST=$(GOCMD) test
 GODEP=$(GOTEST) -i
 GOFMT=gofmt -w
 GOBUILDIMAGE=golang:1.13.4
-GOBUILDPATH=/harbor
+GOBUILDPATHINCONTAINER=/harbor
 
 # go build
 PKG_PATH=github.com/goharbor/harbor/src/pkg
@@ -158,23 +158,24 @@ GOIMAGEBUILDCMD=/usr/local/go/bin/go build -mod vendor
 GOIMAGEBUILD_COMMON=$(GOIMAGEBUILDCMD) $(GOFLAGS) ${GOTAGS} ${GOLDFLAGS}
 GOIMAGEBUILD_CORE=$(GOIMAGEBUILDCMD) $(GOFLAGS) ${GOTAGS} --ldflags "-w -s $(CORE_LDFLAGS)"
 
-GOBUILDPATH_CORE=$(GOBUILDPATH)/src/core
-GOBUILDPATH_JOBSERVICE=$(GOBUILDPATH)/src/jobservice
-GOBUILDPATH_REGISTRYCTL=$(GOBUILDPATH)/src/registryctl
-GOBUILDPATH_MIGRATEPATCH=$(GOBUILDPATH)/src/cmd/migrate-patch
-GOBUILDMAKEPATH=$(GOBUILDPATH)/make
+GOBUILDPATH_CORE=$(GOBUILDPATHINCONTAINER)/src/core
+GOBUILDPATH_JOBSERVICE=$(GOBUILDPATHINCONTAINER)/src/jobservice
+GOBUILDPATH_REGISTRYCTL=$(GOBUILDPATHINCONTAINER)/src/registryctl
+GOBUILDPATH_MIGRATEPATCH=$(GOBUILDPATHINCONTAINER)/src/cmd/migrate-patch
+GOBUILDMAKEPATH=make
 GOBUILDMAKEPATH_CORE=$(GOBUILDMAKEPATH)/photon/core
 GOBUILDMAKEPATH_JOBSERVICE=$(GOBUILDMAKEPATH)/photon/jobservice
 GOBUILDMAKEPATH_REGISTRYCTL=$(GOBUILDMAKEPATH)/photon/registryctl
 GOBUILDMAKEPATH_NOTARY=$(GOBUILDMAKEPATH)/photon/notary
 
 # binary
-CORE_BINARYPATH=$(MAKEDEVPATH)/core
+CORE_BINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_CORE)
 CORE_BINARYNAME=harbor_core
-JOBSERVICEBINARYPATH=$(MAKEDEVPATH)/jobservice
+JOBSERVICEBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_JOBSERVICE)
 JOBSERVICEBINARYNAME=harbor_jobservice
-REGISTRYCTLBINARYPATH=$(MAKEDEVPATH)/registryctl
+REGISTRYCTLBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_REGISTRYCTL)
 REGISTRYCTLBINARYNAME=harbor_registryctl
+MIGRATEPATCHBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_NOTARY)
 MIGRATEPATCHBINARYNAME=migrate-patch
 
 # configfile
@@ -254,14 +255,14 @@ PACKAGE_OFFLINE_PARA=-zcvf harbor-offline-installer-$(PKGVERSIONTAG).tgz \
 					$(HARBORPKG)/prepare \
 					$(HARBORPKG)/LICENSE $(HARBORPKG)/install.sh \
 					$(HARBORPKG)/common.sh \
-					$(HARBORPKG)/harbor.yml
+					$(HARBORPKG)/harbor.yml.tmpl
 
 PACKAGE_ONLINE_PARA=-zcvf harbor-online-installer-$(PKGVERSIONTAG).tgz \
 					$(HARBORPKG)/prepare \
 					$(HARBORPKG)/LICENSE \
 					$(HARBORPKG)/install.sh \
 					$(HARBORPKG)/common.sh \
-					$(HARBORPKG)/harbor.yml
+					$(HARBORPKG)/harbor.yml.tmpl
 
 DOCKERCOMPOSE_FILE_OPT=-f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
 
@@ -288,37 +289,33 @@ check_environment:
 
 compile_core:
 	@echo "compiling binary for core (golang image)..."
-	@echo $(GOBUILDPATH)
-	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_CORE) $(GOBUILDIMAGE) $(GOIMAGEBUILD_CORE) -o $(GOBUILDMAKEPATH_CORE)/$(CORE_BINARYNAME)
+	@echo $(GOBUILDPATHINCONTAINER)
+	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_CORE) $(GOBUILDIMAGE) $(GOIMAGEBUILD_CORE) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_CORE)/$(CORE_BINARYNAME)
 	@echo "Done."
 
 compile_jobservice:
 	@echo "compiling binary for jobservice (golang image)..."
-	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_JOBSERVICE) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDMAKEPATH_JOBSERVICE)/$(JOBSERVICEBINARYNAME)
+	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_JOBSERVICE) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_JOBSERVICE)/$(JOBSERVICEBINARYNAME)
 	@echo "Done."
 
 compile_registryctl:
 	@echo "compiling binary for harbor registry controller (golang image)..."
-	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_REGISTRYCTL) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDMAKEPATH_REGISTRYCTL)/$(REGISTRYCTLBINARYNAME)
+	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_REGISTRYCTL) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_REGISTRYCTL)/$(REGISTRYCTLBINARYNAME)
 	@echo "Done."
 
 compile_notary_migrate_patch:
 	@echo "compiling binary for migrate patch (golang image)..."
-	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_MIGRATEPATCH) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDMAKEPATH_NOTARY)/$(MIGRATEPATCHBINARYNAME)
+	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_MIGRATEPATCH) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_NOTARY)/$(MIGRATEPATCHBINARYNAME)
 	@echo "Done."
 
 compile: check_environment versions_prepare compile_core compile_jobservice compile_registryctl compile_notary_migrate_patch
 
-update_prepare_version:
-	@echo "substitute the prepare version tag in prepare file..."
-	@$(SEDCMD) -i -e 's/goharbor\/prepare:.*[[:space:]]\+/goharbor\/prepare:$(VERSIONTAG) /' $(MAKEPATH)/prepare ;
-
-prepare: update_prepare_version
+prepare:
 	@echo "preparing..."
 	@$(MAKEPATH)/$(PREPARECMD) $(PREPARECMD_PARA)
 
 build:
-	make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) -e GOBUILDIMAGE=$(GOBUILDIMAGE) \
+	make -f $(MAKEFILEPATH_PHOTON)/Makefile $(BUILDTARGET) -e DEVFLAG=$(DEVFLAG) -e GOBUILDIMAGE=$(GOBUILDIMAGE) \
 	 -e REGISTRYVERSION=$(REGISTRYVERSION) -e REGISTRY_SRC_TAG=$(REGISTRY_SRC_TAG) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
 	 -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRADAPTERVERSION=$(CLAIRADAPTERVERSION) -e VERSIONTAG=$(VERSIONTAG) \
 	 -e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \
@@ -334,7 +331,7 @@ build_base_docker:
 
 install: compile build prepare start
 
-package_online: update_prepare_version
+package_online:
 	@echo "packing online package ..."
 	@cp -r make $(HARBORPKG)
 	@if [ -n "$(REGISTRYSERVER)" ] ; then \
@@ -347,7 +344,7 @@ package_online: update_prepare_version
 	@rm -rf $(HARBORPKG)
 	@echo "Done."
 
-package_offline: update_prepare_version compile build
+package_offline: compile build
 
 	@echo "packing offline package ..."
 	@cp -r make $(HARBORPKG)
@@ -462,8 +459,12 @@ swagger_client:
 
 cleanbinary:
 	@echo "cleaning binary..."
-	@if [ -f $(CORE_BINARYPATH)/$(CORE_BINARYNAME) ] ; then rm $(CORE_BINARYPATH)/$(CORE_BINARYNAME) ; fi
-	@if [ -f $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) ] ; then rm $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) ; fi
+	if [ -f $(CORE_BINARYPATH)/$(CORE_BINARYNAME) ] ; then rm $(CORE_BINARYPATH)/$(CORE_BINARYNAME) ; fi
+	if [ -f $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) ] ; then rm $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) ; fi
+	if [ -f $(REGISTRYCTLBINARYPATH)/$(REGISTRYCTLBINARYNAME) ] ; then rm $(REGISTRYCTLBINARYPATH)/$(REGISTRYCTLBINARYNAME) ; fi
+	if [ -f $(MIGRATEPATCHBINARYPATH)/$(MIGRATEPATCHBINARYNAME) ] ; then rm $(MIGRATEPATCHBINARYPATH)/$(MIGRATEPATCHBINARYNAME) ; fi
+	rm -rf make/photon/*/binary/
+
 
 cleanimage:
 	@echo "cleaning image for photon..."
@@ -485,8 +486,18 @@ cleanpackage:
 	@if [ -f $(BUILDPATH)/harbor-offline-installer-$(VERSIONTAG).tgz ] ; \
 	then rm $(BUILDPATH)/harbor-offline-installer-$(VERSIONTAG).tgz ; fi
 
+cleanconfig:
+	@echo "clean generated config files"
+	rm -f $(BUILDPATH)/make/photon/prepare/versions
+	rm -f $(BUILDPATH)/UIVERSION
+	rm -rf $(BUILDPATH)/make/common
+	rm -rf $(BUILDPATH)/harborclient
+	rm -rf $(BUILDPATH)/src/portal/dist
+	rm -rf $(BUILDPATH)/src/portal/lib/dist
+	rm -f $(BUILDPATH)/src/portal/proxy.config.json
+
 .PHONY: cleanall
-cleanall: cleanbinary cleanimage cleandockercomposefile cleanpackage
+cleanall: cleanbinary cleanimage cleandockercomposefile cleanconfig cleanpackage
 
 clean:
 	@echo "  make cleanall:		remove binary, Harbor images, specific version docker-compose"

make/photon/Makefile

@@ -9,7 +9,6 @@
 SHELL := /bin/bash
 BUILDPATH=$(CURDIR)
 MAKEPATH=$(BUILDPATH)/make
-MAKEDEVPATH=$(MAKEPATH)/dev
 SRCPATH=./src
 TOOLSPATH=$(CURDIR)/tools
 SEDCMD=$(shell which sed)
@@ -23,10 +22,8 @@ DOCKERIMASES=$(DOCKERCMD) images
 
 # binary
 CORE_SOURCECODE=$(SRCPATH)/core
-CORE_BINARYPATH=$(MAKEDEVPATH)/core
 CORE_BINARYNAME=harbor_core
 JOBSERVICESOURCECODE=$(SRCPATH)/jobservice
-JOBSERVICEBINARYPATH=$(MAKEDEVPATH)/jobservice
 JOBSERVICEBINARYNAME=harbor_jobservice
 
 # photon dockerfile

make/photon/portal/Dockerfile

@@ -4,7 +4,6 @@ FROM node:10.15.0 as nodeportal
 WORKDIR /build_dir
 
 ARG npm_registry=https://registry.npmjs.org
-ENV NPM_CONFIG_REGISTRY=${npm_registry}
 
 COPY src/portal/package.json /build_dir
 COPY src/portal/package-lock.json /build_dir
@@ -18,6 +17,8 @@ RUN apt-get update \
 COPY ./LICENSE /build_dir
 COPY src/portal /build_dir
 
+ENV NPM_CONFIG_REGISTRY=${npm_registry}
+
 RUN node --max_old_space_size=2048 'node_modules/@angular/cli/bin/ng' build --prod
 
 FROM goharbor/harbor-portal-base:${harbor_base_image_version}

make/photon/prepare/templates/nginx/nginx.https.conf.jinja

@@ -162,6 +162,6 @@ http {
   server {
       listen 8080;
       #server_name harbordomain.com;
-      return 308 https://$host$request_uri;
+      return 308 https://{{https_redirect}}$request_uri;
   } 
 }

make/photon/prepare/utils/nginx.py

@@ -59,6 +59,7 @@ def render_nginx_template(config_dict):
             nginx_conf,
             uid=DEFAULT_UID,
             gid=DEFAULT_GID,
+            https_redirect='$host' + ('https_port' in config_dict and (":" + str(config_dict['https_port'])) or ""),
             ssl_cert=SSL_CERT_PATH,
             ssl_cert_key=SSL_CERT_KEY_PATH)
         location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTPS

make/prepare

@@ -1,5 +1,5 @@
 #!/bin/bash
-set +e
+set -e
 
 # If compiling source code this dir is harbor's make dir.
 # If installing harbor via pacakge, this dir is harbor's root dir.
@@ -9,7 +9,30 @@ else
     harbor_prepare_path="$( cd "$(dirname "$0")" ; pwd -P )"
 fi
 echo "prepare base dir is set to ${harbor_prepare_path}"
-data_path=$(grep '^[^#]*data_volume:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}')
+
+# Clean up input dir
+rm -rf ${harbor_prepare_path}/input
+# Create a input dirs
+mkdir -p ${harbor_prepare_path}/input
+input_dir=${harbor_prepare_path}/input
+
+# Copy harbor.yml to input dir
+if [[ ! "$1" =~ ^\-\- ]] && [ -f "$1" ]
+then
+    cp $1 $input_dir/harbor.yml
+    shift
+else
+    if [ -f "${harbor_prepare_path}/harbor.yml" ];then
+        cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml
+    else
+        echo "no config file: ${harbor_prepare_path}/harbor.yml"
+        exit 1
+    fi
+fi
+
+data_path=$(grep '^[^#]*data_volume:' $input_dir/harbor.yml | awk '{print $NF}')
+
+echo "VERSIONTAG:${VERSIONTAG:=dev}"
 
 # If previous secretkeys exist, move it to new location
 previous_secretkey_path=/data/secretkey
@@ -24,21 +47,6 @@ if [ -f $previous_defaultalias_path ]; then
     mv $previous_defaultalias_path $data_path/secret/keys
 fi
 
-# Clean up input dir
-rm -rf ${harbor_prepare_path}/input
-# Create a input dirs
-mkdir -p ${harbor_prepare_path}/input
-input_dir=${harbor_prepare_path}/input
-
-set -e
-
-# Copy harbor.yml to input dir
-if [[ ! "$1" =~ ^\-\- ]] && [ -f "$1" ]
-then
-    cp $1 $input_dir/harbor.yml
-else
-    cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml
-fi
 
 # Create secret dir
 secret_dir=${data_path}/secret
@@ -51,7 +59,7 @@ docker run --rm -v $input_dir:/input:z \
                     -v $config_dir:/config:z \
                     -v $secret_dir:/secret:z \
                     -v /:/hostfs:z \
-                    goharbor/prepare:dev $@
+                    goharbor/prepare:${VERSIONTAG} $@
 
 echo "Clean up the input dir"
 # Clean up input dir

tests/.gitignore

@@ -0,0 +1,4 @@
+*.crt
+*.key
+*.csr
+extfile.cnf

tests/hostcfg.sh

@@ -2,7 +2,7 @@
 IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'`
 
 #echo $IP
-sudo sed "s/reg.mydomain.com/$IP/" -i make/harbor.yml
+sudo sed "s/reg.mydomain.com/$IP/" make/harbor.yml.tmpl |sudo tee make/harbor.yml
 
 # TODO: remove it when scanner adapter support internal access of harbor
 echo "storage_service:" >> make/harbor.yml

tests/showtime.sh

@@ -0,0 +1,13 @@
+#/bin/env bash
+
+if [ ! -z "$*" ]; then
+ $@ 2>&1 | while read line;do
+  echo $(date +"%T") $line
+ done
+ exit ${PIPESTATUS[0]}
+else
+ while read line;do
+   echo $(date +"%T") $line
+  done
+  echo ret $?
+fi

tests/travis/api_common_install.sh

@@ -1,13 +1,14 @@
 #!/bin/bash
+set -x
 
 set +e
 sudo rm -fr /data/*
 sudo mkdir -p /data
 
 set -e
+if [ -z "$1" ]; then echo no ip specified; exit 1;fi
 # prepare cert ...
-sudo sed "s/127.0.0.1/$1/" -i tests/generateCerts.sh
-sudo ./tests/generateCerts.sh
+sudo ./tests/generateCerts.sh $1
 sudo mkdir -p /etc/docker/certs.d/$1 && sudo cp ./tests/harbor_ca.crt /etc/docker/certs.d/$1/ && rm -rf ~/.docker/ &&  mkdir -p ~/.docker/tls/$1:4443/ && sudo cp ./tests/harbor_ca.crt ~/.docker/tls/$1:4443/
 
 sudo ./tests/hostcfg.sh

tests/travis/api_run.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -x
 
 #source gskey.sh
 

tests/travis/distro_installer.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -x
 
 set -e
 

tests/travis/ui_ut_run.sh

@@ -1,3 +1,7 @@
+#!/bin/bash
+set -x
+set -e
+
 cd ./src/portal
 npm install -g -q --no-progress angular-cli
 npm install -g -q --no-progress karma

tests/travis/ut_install.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -x
 
 set -e
 

tests/travis/ut_run.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -x
 
 set -e