Merge pull request #10047 from bitsf/makefile_clean

optimize the makefile process
This commit is contained in:
Wang Yan 2019-12-05 19:03:19 +08:00 committed by GitHub
commit 2a63382236
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 110 additions and 63 deletions

1
.gitignore vendored
View File

@ -1,5 +1,6 @@
!/contrib/helm/harbor
make/harbor.yml
make/docker-compose.yml
make/common/config/*
make/dev/adminserver/harbor_adminserver

View File

@ -38,7 +38,7 @@ env:
- CORE_SECRET: tempString
- KEY_PATH: "/data/secret/keys/secretkey"
- REDIS_HOST: localhost
- REG_VERSION: v2.7.1
- REG_VERSION: v2.7.1-patch-2819-2553
- UI_BUILDER_VERSION: 1.6.0
- TOKEN_PRIVATE_KEY_PATH: "/home/travis/gopath/src/github.com/goharbor/harbor/tests/private_key.pem"
addons:
@ -56,16 +56,18 @@ before_install:
- IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'`
- sudo sed -i '$a DOCKER_OPTS=\"--insecure-registry '$IP':5000\"' /etc/default/docker
- export IP=$IP
- sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
- sudo update-ca-certificates
- sudo service docker restart
install:
- if [ "$UTTEST" == true ]; then bash ./tests/travis/ut_install.sh ; fi
- if [ "$APITEST_DB" == true ]; then bash ./tests/travis/api_common_install.sh $IP
- if [ "$UTTEST" == true ]; then bash ./tests/showtime.sh ./tests/travis/ut_install.sh ; fi
- if [ "$APITEST_DB" == true ]; then bash ./tests/showtime.sh ./tests/travis/api_common_install.sh $IP
DB; fi
- if [ "$APITEST_LDAP" == true ]; then bash ./tests/travis/api_common_install.sh $IP
- if [ "$APITEST_LDAP" == true ]; then bash ./tests/showtime.sh ./tests/travis/api_common_install.sh $IP
LDAP; fi
script:
- if [ "$UTTEST" == true ]; then bash ./tests/travis/ut_run.sh $IP; fi
- if [ "$APITEST_DB" == true ]; then bash ./tests/travis/api_run.sh DB $IP; fi
- if [ "$APITEST_LDAP" == true ]; then bash ./tests/travis/api_run.sh LDAP $IP; fi
- if [ "$OFFLINE" == true ]; then bash ./tests/travis/distro_installer.sh; fi
- if [ "$UI_UT" == true ]; then bash ./tests/travis/ui_ut_run.sh ; fi
- if [ "$UTTEST" == true ]; then bash ./tests/showtime.sh ./tests/travis/ut_run.sh $IP; fi
- if [ "$APITEST_DB" == true ]; then bash ./tests/showtime.sh ./tests/travis/api_run.sh DB $IP; fi
- if [ "$APITEST_LDAP" == true ]; then bash ./tests/showtime.sh ./tests/travis/api_run.sh LDAP $IP; fi
- if [ "$OFFLINE" == true ]; then bash ./tests/showtime.sh ./tests/travis/distro_installer.sh; fi
- if [ "$UI_UT" == true ]; then bash ./tests/showtime.sh ./tests/travis/ui_ut_run.sh ; fi

View File

@ -64,7 +64,6 @@
SHELL := /bin/bash
BUILDPATH=$(CURDIR)
MAKEPATH=$(BUILDPATH)/make
MAKEDEVPATH=$(MAKEPATH)/dev
MAKE_PREPARE_PATH=$(MAKEPATH)/photon/prepare
SRCPATH=./src
TOOLSPATH=$(BUILDPATH)/tools
@ -84,6 +83,7 @@ MIGRATORFLAG=false
NPM_REGISTRY=https://registry.npmjs.org
# enable/disable chart repo supporting
CHARTFLAG=false
BUILDTARGET=build
# version prepare
# for docker image tag
@ -139,7 +139,7 @@ GOTEST=$(GOCMD) test
GODEP=$(GOTEST) -i
GOFMT=gofmt -w
GOBUILDIMAGE=golang:1.13.4
GOBUILDPATH=/harbor
GOBUILDPATHINCONTAINER=/harbor
# go build
PKG_PATH=github.com/goharbor/harbor/src/pkg
@ -158,23 +158,24 @@ GOIMAGEBUILDCMD=/usr/local/go/bin/go build -mod vendor
GOIMAGEBUILD_COMMON=$(GOIMAGEBUILDCMD) $(GOFLAGS) ${GOTAGS} ${GOLDFLAGS}
GOIMAGEBUILD_CORE=$(GOIMAGEBUILDCMD) $(GOFLAGS) ${GOTAGS} --ldflags "-w -s $(CORE_LDFLAGS)"
GOBUILDPATH_CORE=$(GOBUILDPATH)/src/core
GOBUILDPATH_JOBSERVICE=$(GOBUILDPATH)/src/jobservice
GOBUILDPATH_REGISTRYCTL=$(GOBUILDPATH)/src/registryctl
GOBUILDPATH_MIGRATEPATCH=$(GOBUILDPATH)/src/cmd/migrate-patch
GOBUILDMAKEPATH=$(GOBUILDPATH)/make
GOBUILDPATH_CORE=$(GOBUILDPATHINCONTAINER)/src/core
GOBUILDPATH_JOBSERVICE=$(GOBUILDPATHINCONTAINER)/src/jobservice
GOBUILDPATH_REGISTRYCTL=$(GOBUILDPATHINCONTAINER)/src/registryctl
GOBUILDPATH_MIGRATEPATCH=$(GOBUILDPATHINCONTAINER)/src/cmd/migrate-patch
GOBUILDMAKEPATH=make
GOBUILDMAKEPATH_CORE=$(GOBUILDMAKEPATH)/photon/core
GOBUILDMAKEPATH_JOBSERVICE=$(GOBUILDMAKEPATH)/photon/jobservice
GOBUILDMAKEPATH_REGISTRYCTL=$(GOBUILDMAKEPATH)/photon/registryctl
GOBUILDMAKEPATH_NOTARY=$(GOBUILDMAKEPATH)/photon/notary
# binary
CORE_BINARYPATH=$(MAKEDEVPATH)/core
CORE_BINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_CORE)
CORE_BINARYNAME=harbor_core
JOBSERVICEBINARYPATH=$(MAKEDEVPATH)/jobservice
JOBSERVICEBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_JOBSERVICE)
JOBSERVICEBINARYNAME=harbor_jobservice
REGISTRYCTLBINARYPATH=$(MAKEDEVPATH)/registryctl
REGISTRYCTLBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_REGISTRYCTL)
REGISTRYCTLBINARYNAME=harbor_registryctl
MIGRATEPATCHBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_NOTARY)
MIGRATEPATCHBINARYNAME=migrate-patch
# configfile
@ -254,14 +255,14 @@ PACKAGE_OFFLINE_PARA=-zcvf harbor-offline-installer-$(PKGVERSIONTAG).tgz \
$(HARBORPKG)/prepare \
$(HARBORPKG)/LICENSE $(HARBORPKG)/install.sh \
$(HARBORPKG)/common.sh \
$(HARBORPKG)/harbor.yml
$(HARBORPKG)/harbor.yml.tmpl
PACKAGE_ONLINE_PARA=-zcvf harbor-online-installer-$(PKGVERSIONTAG).tgz \
$(HARBORPKG)/prepare \
$(HARBORPKG)/LICENSE \
$(HARBORPKG)/install.sh \
$(HARBORPKG)/common.sh \
$(HARBORPKG)/harbor.yml
$(HARBORPKG)/harbor.yml.tmpl
DOCKERCOMPOSE_FILE_OPT=-f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
@ -288,37 +289,33 @@ check_environment:
compile_core:
@echo "compiling binary for core (golang image)..."
@echo $(GOBUILDPATH)
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_CORE) $(GOBUILDIMAGE) $(GOIMAGEBUILD_CORE) -o $(GOBUILDMAKEPATH_CORE)/$(CORE_BINARYNAME)
@echo $(GOBUILDPATHINCONTAINER)
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_CORE) $(GOBUILDIMAGE) $(GOIMAGEBUILD_CORE) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_CORE)/$(CORE_BINARYNAME)
@echo "Done."
compile_jobservice:
@echo "compiling binary for jobservice (golang image)..."
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_JOBSERVICE) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDMAKEPATH_JOBSERVICE)/$(JOBSERVICEBINARYNAME)
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_JOBSERVICE) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_JOBSERVICE)/$(JOBSERVICEBINARYNAME)
@echo "Done."
compile_registryctl:
@echo "compiling binary for harbor registry controller (golang image)..."
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_REGISTRYCTL) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDMAKEPATH_REGISTRYCTL)/$(REGISTRYCTLBINARYNAME)
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_REGISTRYCTL) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_REGISTRYCTL)/$(REGISTRYCTLBINARYNAME)
@echo "Done."
compile_notary_migrate_patch:
@echo "compiling binary for migrate patch (golang image)..."
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_MIGRATEPATCH) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDMAKEPATH_NOTARY)/$(MIGRATEPATCHBINARYNAME)
@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_MIGRATEPATCH) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_NOTARY)/$(MIGRATEPATCHBINARYNAME)
@echo "Done."
compile: check_environment versions_prepare compile_core compile_jobservice compile_registryctl compile_notary_migrate_patch
update_prepare_version:
@echo "substitute the prepare version tag in prepare file..."
@$(SEDCMD) -i -e 's/goharbor\/prepare:.*[[:space:]]\+/goharbor\/prepare:$(VERSIONTAG) /' $(MAKEPATH)/prepare ;
prepare: update_prepare_version
prepare:
@echo "preparing..."
@$(MAKEPATH)/$(PREPARECMD) $(PREPARECMD_PARA)
build:
make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) -e GOBUILDIMAGE=$(GOBUILDIMAGE) \
make -f $(MAKEFILEPATH_PHOTON)/Makefile $(BUILDTARGET) -e DEVFLAG=$(DEVFLAG) -e GOBUILDIMAGE=$(GOBUILDIMAGE) \
-e REGISTRYVERSION=$(REGISTRYVERSION) -e REGISTRY_SRC_TAG=$(REGISTRY_SRC_TAG) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
-e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRADAPTERVERSION=$(CLAIRADAPTERVERSION) -e VERSIONTAG=$(VERSIONTAG) \
-e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \
@ -334,7 +331,7 @@ build_base_docker:
install: compile build prepare start
package_online: update_prepare_version
package_online:
@echo "packing online package ..."
@cp -r make $(HARBORPKG)
@if [ -n "$(REGISTRYSERVER)" ] ; then \
@ -347,7 +344,7 @@ package_online: update_prepare_version
@rm -rf $(HARBORPKG)
@echo "Done."
package_offline: update_prepare_version compile build
package_offline: compile build
@echo "packing offline package ..."
@cp -r make $(HARBORPKG)
@ -462,8 +459,12 @@ swagger_client:
cleanbinary:
@echo "cleaning binary..."
@if [ -f $(CORE_BINARYPATH)/$(CORE_BINARYNAME) ] ; then rm $(CORE_BINARYPATH)/$(CORE_BINARYNAME) ; fi
@if [ -f $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) ] ; then rm $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) ; fi
if [ -f $(CORE_BINARYPATH)/$(CORE_BINARYNAME) ] ; then rm $(CORE_BINARYPATH)/$(CORE_BINARYNAME) ; fi
if [ -f $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) ] ; then rm $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) ; fi
if [ -f $(REGISTRYCTLBINARYPATH)/$(REGISTRYCTLBINARYNAME) ] ; then rm $(REGISTRYCTLBINARYPATH)/$(REGISTRYCTLBINARYNAME) ; fi
if [ -f $(MIGRATEPATCHBINARYPATH)/$(MIGRATEPATCHBINARYNAME) ] ; then rm $(MIGRATEPATCHBINARYPATH)/$(MIGRATEPATCHBINARYNAME) ; fi
rm -rf make/photon/*/binary/
cleanimage:
@echo "cleaning image for photon..."
@ -485,8 +486,18 @@ cleanpackage:
@if [ -f $(BUILDPATH)/harbor-offline-installer-$(VERSIONTAG).tgz ] ; \
then rm $(BUILDPATH)/harbor-offline-installer-$(VERSIONTAG).tgz ; fi
cleanconfig:
@echo "clean generated config files"
rm -f $(BUILDPATH)/make/photon/prepare/versions
rm -f $(BUILDPATH)/UIVERSION
rm -rf $(BUILDPATH)/make/common
rm -rf $(BUILDPATH)/harborclient
rm -rf $(BUILDPATH)/src/portal/dist
rm -rf $(BUILDPATH)/src/portal/lib/dist
rm -f $(BUILDPATH)/src/portal/proxy.config.json
.PHONY: cleanall
cleanall: cleanbinary cleanimage cleandockercomposefile cleanpackage
cleanall: cleanbinary cleanimage cleandockercomposefile cleanconfig cleanpackage
clean:
@echo " make cleanall: remove binary, Harbor images, specific version docker-compose"

View File

@ -9,7 +9,6 @@
SHELL := /bin/bash
BUILDPATH=$(CURDIR)
MAKEPATH=$(BUILDPATH)/make
MAKEDEVPATH=$(MAKEPATH)/dev
SRCPATH=./src
TOOLSPATH=$(CURDIR)/tools
SEDCMD=$(shell which sed)
@ -23,10 +22,8 @@ DOCKERIMASES=$(DOCKERCMD) images
# binary
CORE_SOURCECODE=$(SRCPATH)/core
CORE_BINARYPATH=$(MAKEDEVPATH)/core
CORE_BINARYNAME=harbor_core
JOBSERVICESOURCECODE=$(SRCPATH)/jobservice
JOBSERVICEBINARYPATH=$(MAKEDEVPATH)/jobservice
JOBSERVICEBINARYNAME=harbor_jobservice
# photon dockerfile

View File

@ -4,7 +4,6 @@ FROM node:10.15.0 as nodeportal
WORKDIR /build_dir
ARG npm_registry=https://registry.npmjs.org
ENV NPM_CONFIG_REGISTRY=${npm_registry}
COPY src/portal/package.json /build_dir
COPY src/portal/package-lock.json /build_dir
@ -18,6 +17,8 @@ RUN apt-get update \
COPY ./LICENSE /build_dir
COPY src/portal /build_dir
ENV NPM_CONFIG_REGISTRY=${npm_registry}
RUN node --max_old_space_size=2048 'node_modules/@angular/cli/bin/ng' build --prod
FROM goharbor/harbor-portal-base:${harbor_base_image_version}

View File

@ -162,6 +162,6 @@ http {
server {
listen 8080;
#server_name harbordomain.com;
return 308 https://$host$request_uri;
return 308 https://{{https_redirect}}$request_uri;
}
}

View File

@ -59,6 +59,7 @@ def render_nginx_template(config_dict):
nginx_conf,
uid=DEFAULT_UID,
gid=DEFAULT_GID,
https_redirect='$host' + ('https_port' in config_dict and (":" + str(config_dict['https_port'])) or ""),
ssl_cert=SSL_CERT_PATH,
ssl_cert_key=SSL_CERT_KEY_PATH)
location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTPS

View File

@ -1,5 +1,5 @@
#!/bin/bash
set +e
set -e
# If compiling source code this dir is harbor's make dir.
# If installing harbor via pacakge, this dir is harbor's root dir.
@ -9,7 +9,30 @@ else
harbor_prepare_path="$( cd "$(dirname "$0")" ; pwd -P )"
fi
echo "prepare base dir is set to ${harbor_prepare_path}"
data_path=$(grep '^[^#]*data_volume:' ${harbor_prepare_path}/harbor.yml | awk '{print $NF}')
# Clean up input dir
rm -rf ${harbor_prepare_path}/input
# Create a input dirs
mkdir -p ${harbor_prepare_path}/input
input_dir=${harbor_prepare_path}/input
# Copy harbor.yml to input dir
if [[ ! "$1" =~ ^\-\- ]] && [ -f "$1" ]
then
cp $1 $input_dir/harbor.yml
shift
else
if [ -f "${harbor_prepare_path}/harbor.yml" ];then
cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml
else
echo "no config file: ${harbor_prepare_path}/harbor.yml"
exit 1
fi
fi
data_path=$(grep '^[^#]*data_volume:' $input_dir/harbor.yml | awk '{print $NF}')
echo "VERSIONTAG:${VERSIONTAG:=dev}"
# If previous secretkeys exist, move it to new location
previous_secretkey_path=/data/secretkey
@ -24,21 +47,6 @@ if [ -f $previous_defaultalias_path ]; then
mv $previous_defaultalias_path $data_path/secret/keys
fi
# Clean up input dir
rm -rf ${harbor_prepare_path}/input
# Create a input dirs
mkdir -p ${harbor_prepare_path}/input
input_dir=${harbor_prepare_path}/input
set -e
# Copy harbor.yml to input dir
if [[ ! "$1" =~ ^\-\- ]] && [ -f "$1" ]
then
cp $1 $input_dir/harbor.yml
else
cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml
fi
# Create secret dir
secret_dir=${data_path}/secret
@ -51,7 +59,7 @@ docker run --rm -v $input_dir:/input:z \
-v $config_dir:/config:z \
-v $secret_dir:/secret:z \
-v /:/hostfs:z \
goharbor/prepare:dev $@
goharbor/prepare:${VERSIONTAG} $@
echo "Clean up the input dir"
# Clean up input dir

4
tests/.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
*.crt
*.key
*.csr
extfile.cnf

View File

@ -2,7 +2,7 @@
IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'`
#echo $IP
sudo sed "s/reg.mydomain.com/$IP/" -i make/harbor.yml
sudo sed "s/reg.mydomain.com/$IP/" make/harbor.yml.tmpl |sudo tee make/harbor.yml
# TODO: remove it when scanner adapter support internal access of harbor
echo "storage_service:" >> make/harbor.yml

13
tests/showtime.sh Executable file
View File

@ -0,0 +1,13 @@
#/bin/env bash
if [ ! -z "$*" ]; then
$@ 2>&1 | while read line;do
echo $(date +"%T") $line
done
exit ${PIPESTATUS[0]}
else
while read line;do
echo $(date +"%T") $line
done
echo ret $?
fi

5
tests/travis/api_common_install.sh Normal file → Executable file
View File

@ -1,13 +1,14 @@
#!/bin/bash
set -x
set +e
sudo rm -fr /data/*
sudo mkdir -p /data
set -e
if [ -z "$1" ]; then echo no ip specified; exit 1;fi
# prepare cert ...
sudo sed "s/127.0.0.1/$1/" -i tests/generateCerts.sh
sudo ./tests/generateCerts.sh
sudo ./tests/generateCerts.sh $1
sudo mkdir -p /etc/docker/certs.d/$1 && sudo cp ./tests/harbor_ca.crt /etc/docker/certs.d/$1/ && rm -rf ~/.docker/ && mkdir -p ~/.docker/tls/$1:4443/ && sudo cp ./tests/harbor_ca.crt ~/.docker/tls/$1:4443/
sudo ./tests/hostcfg.sh

1
tests/travis/api_run.sh Normal file → Executable file
View File

@ -1,4 +1,5 @@
#!/bin/bash
set -x
#source gskey.sh

View File

@ -1,4 +1,5 @@
#!/bin/bash
set -x
set -e

4
tests/travis/ui_ut_run.sh Normal file → Executable file
View File

@ -1,3 +1,7 @@
#!/bin/bash
set -x
set -e
cd ./src/portal
npm install -g -q --no-progress angular-cli
npm install -g -q --no-progress karma

1
tests/travis/ut_install.sh Normal file → Executable file
View File

@ -1,4 +1,5 @@
#!/bin/bash
set -x
set -e

View File

@ -1,4 +1,5 @@
#!/bin/bash
set -x
set -e