Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-22 10:15:35 +01:00
Code Issues Projects Releases Wiki Activity

enhance[cicd] introduce github action for CICD

Browse Source 
In order to replace travis.
Implement 5 CI jobs
- UTTEST
- APITEST_DB
- APITEST_LDAP
- OFFLINE
- UI_UT

Signed-off-by: Ziming Zhang <zziming@vmware.com>
This commit is contained in:
Ziming 2019-12-11 19:11:20 +08:00 committed by Ziming Zhang
parent 92a0206ca8
commit e32649adb4
13 changed files with 309 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

277
.github/workflows/CI.yml vendored Normal file
View File

@ -0,0 +1,277 @@
name: CI
env:
POSTGRESQL_HOST: localhost
POSTGRESQL_PORT: 5432
POSTGRESQL_USR: postgres
POSTGRESQL_PWD: root123
POSTGRESQL_DATABASE: registry
ADMINSERVER_URL: http://127.0.0.1:8888
DOCKER_COMPOSE_VERSION: 1.23.0
HARBOR_ADMIN: admin
HARBOR_ADMIN_PASSWD: Harbor12345
CORE_SECRET: tempString
KEY_PATH: "/data/secret/keys/secretkey"
REDIS_HOST: localhost
REG_VERSION: v2.7.1-patch-2819-2553
UI_BUILDER_VERSION: 1.6.0
on:
pull_request:
push:
jobs:
UTTEST:
env:
UTTEST: true
runs-on:
#- self-hosted
- ubuntu-latest
steps:
- name: Set up Go 1.13
uses: actions/setup-go@v1
with:
go-version: 1.13
id: go
- name: setup Docker
uses: docker-practice/actions-setup-docker@0.0.1
with:
docker_version: 18.09
docker_channel: stable
- uses: actions/checkout@v1
with:
fetch-depth: 1
path: src/github.com/goharbor/harbor
- name: setup env
run: |
pwd
go env
echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
echo "::add-path::$(go env GOPATH)/bin"
echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
shell: bash
- name: before_install
run: |
set -x
pwd
env
#sudo apt install -y xvfb
#xvfb-run ls
curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
chmod +x docker-compose
sudo mv docker-compose /usr/local/bin
IP=`hostname -I | awk '{print $1}'`
echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
echo "::set-env name=IP::$IP"
sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
sudo update-ca-certificates
sudo service docker restart
- name: install
run: |
env
bash ./tests/showtime.sh ./tests/travis/ut_install.sh
- name: script
run: |
echo IP: $IP
bash ./tests/showtime.sh ./tests/travis/ut_run.sh $IP
APITEST_DB:
env:
APITEST_DB: true
runs-on:
#- self-hosted
- ubuntu-latest
steps:
- name: Set up Go 1.13
uses: actions/setup-go@v1
with:
go-version: 1.13
id: go
- name: setup Docker
uses: docker-practice/actions-setup-docker@0.0.1
with:
docker_version: 18.09
docker_channel: stable
- uses: actions/checkout@v1
with:
fetch-depth: 1
path: src/github.com/goharbor/harbor
- name: setup env
run: |
pwd
go env
echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
echo "::add-path::$(go env GOPATH)/bin"
echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
shell: bash
- name: before_install
run: |
set -x
pwd
env
#sudo apt install -y xvfb
#xvfb-run ls
curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
chmod +x docker-compose
sudo mv docker-compose /usr/local/bin
IP=`hostname -I | awk '{print $1}'`
echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
echo "::set-env name=IP::$IP"
sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
sudo update-ca-certificates
sudo service docker restart
- name: install
run: |
env
bash ./tests/showtime.sh ./tests/travis/api_common_install.sh $IP DB
- name: script
run: |
echo IP: $IP
bash ./tests/showtime.sh ./tests/travis/api_run.sh DB $IP
APITEST_LDAP:
env:
APITEST_LDAP: true
runs-on:
#- self-hosted
- ubuntu-latest
steps:
- name: Set up Go 1.13
uses: actions/setup-go@v1
with:
go-version: 1.13
id: go
- name: setup Docker
uses: docker-practice/actions-setup-docker@0.0.1
with:
docker_version: 18.09
docker_channel: stable
- uses: actions/checkout@v1
with:
fetch-depth: 1
path: src/github.com/goharbor/harbor
- name: setup env
run: |
pwd
go env
echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
echo "::add-path::$(go env GOPATH)/bin"
echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
shell: bash
- name: before_install
run: |
set -x
pwd
env
#sudo apt install -y xvfb
#xvfb-run ls
curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
chmod +x docker-compose
sudo mv docker-compose /usr/local/bin
IP=`hostname -I | awk '{print $1}'`
echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
echo "::set-env name=IP::$IP"
sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
sudo update-ca-certificates
sudo service docker restart
- name: install
run: |
env
bash ./tests/showtime.sh ./tests/travis/api_common_install.sh $IP LDAP
- name: script
run: |
echo IP: $IP
bash ./tests/showtime.sh ./tests/travis/api_run.sh LDAP $IP
OFFLINE:
env:
OFFLINE: true
runs-on:
#- self-hosted
- ubuntu-latest
steps:
- name: Set up Go 1.13
uses: actions/setup-go@v1
with:
go-version: 1.13
id: go
- name: setup Docker
uses: docker-practice/actions-setup-docker@0.0.1
with:
docker_version: 18.09
docker_channel: stable
- uses: actions/checkout@v1
with:
fetch-depth: 1
path: src/github.com/goharbor/harbor
- name: setup env
run: |
pwd
docker version
go env
echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
echo "::add-path::$(go env GOPATH)/bin"
echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
shell: bash
- name: before_install
run: |
set -x
pwd
env
#sudo apt install -y xvfb
#xvfb-run ls
curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
chmod +x docker-compose
sudo mv docker-compose /usr/local/bin
IP=`hostname -I | awk '{print $1}'`
echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
echo "::set-env name=IP::$IP"
sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
sudo update-ca-certificates
sudo service docker restart
- name: script
run: |
echo IP: $IP
bash ./tests/showtime.sh ./tests/travis/distro_installer.sh
UI_UT:
env:
UI_UT: true
runs-on:
#- self-hosted
- ubuntu-latest
steps:
- uses: actions/setup-node@v1
with:
node-version: '10.16.2'
- uses: actions/checkout@v1
with:
fetch-depth: 1
path: src/github.com/goharbor/harbor
- name: setup env
run: |
pwd
go env
echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
echo "::add-path::$(go env GOPATH)/bin"
echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
shell: bash
- name: before_install
run: |
set -x
pwd
env
#sudo apt install -y xvfb
#xvfb-run ls
#curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
#chmod +x docker-compose
#sudo mv docker-compose /usr/local/bin
IP=`hostname -I | awk '{print $1}'`
#echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
echo "::set-env name=IP::$IP"
sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
sudo update-ca-certificates
#sudo service docker restart
- name: script
run: |
echo IP: $IP
bash ./tests/showtime.sh ./tests/travis/ui_ut_run.sh

8
make/common.sh
View File

@ -89,13 +89,12 @@ function check_docker {
docker_version_part1=${BASH_REMATCH[2]} docker_version_part1=${BASH_REMATCH[2]}
docker_version_part2=${BASH_REMATCH[3]} docker_version_part2=${BASH_REMATCH[3]}
note "docker version: $docker_version"
# the version of docker does not meet the requirement # the version of docker does not meet the requirement
if [ "$docker_version_part1" -lt 17 ] || ([ "$docker_version_part1" -eq 17 ] && [ "$docker_version_part2" -lt 6 ]) if [ "$docker_version_part1" -lt 17 ] || ([ "$docker_version_part1" -eq 17 ] && [ "$docker_version_part2" -lt 6 ])
then then
error "Need to upgrade docker package to 17.06.0+." error "Need to upgrade docker package to 17.06.0+."
exit 1 exit 1
else
note "docker version: $docker_version"
fi fi
else else
error "Failed to parse docker version." error "Failed to parse docker version."
@ -117,13 +116,12 @@ function check_dockercompose {
docker_compose_version_part1=${BASH_REMATCH[2]} docker_compose_version_part1=${BASH_REMATCH[2]}
docker_compose_version_part2=${BASH_REMATCH[3]} docker_compose_version_part2=${BASH_REMATCH[3]}
note "docker-compose version: $docker_compose_version"
# the version of docker-compose does not meet the requirement # the version of docker-compose does not meet the requirement
if [ "$docker_compose_version_part1" -lt 1 ] || ([ "$docker_compose_version_part1" -eq 1 ] && [ "$docker_compose_version_part2" -lt 18 ]) if [ "$docker_compose_version_part1" -lt 1 ] || ([ "$docker_compose_version_part1" -eq 1 ] && [ "$docker_compose_version_part2" -lt 18 ])
then then
error "Need to upgrade docker-compose package to 1.18.0+." error "Need to upgrade docker-compose package to 1.18.0+."
exit 1 exit 1
else
note "docker-compose version: $docker_compose_version"
fi fi
else else
error "Failed to parse docker-compose version." error "Failed to parse docker-compose version."

2
make/photon/registry/Dockerfile
View File

@ -16,5 +16,5 @@ USER harbor
ENTRYPOINT ["/home/harbor/entrypoint.sh"] ENTRYPOINT ["/home/harbor/entrypoint.sh"]
VOLUME ["/var/lib/registry"] VOLUME ["/storage"]
EXPOSE 5000 EXPOSE 5000

2
src/core/api/api_test.go
View File

@ -174,7 +174,7 @@ func runCodeCheckingCases(t *testing.T, cases ...*codeCheckingCase) {
if resp.Body.Len() > 0 { if resp.Body.Len() > 0 {
t.Log(resp.Body.String()) t.Log(resp.Body.String())
} }
continue t.FailNow()
} }
if c.postFunc != nil { if c.postFunc != nil {

4
src/core/api/utils.go
View File

@ -64,7 +64,7 @@ func SyncRegistry(pm promgr.ProjectManager) error {
} }
if len(reposToAdd) > 0 { if len(reposToAdd) > 0 {
log.Debugf("Start adding repositories into DB... ") log.Infof("Start adding repositories into DB %v ... ", len(reposToAdd))
for _, repoToAdd := range reposToAdd { for _, repoToAdd := range reposToAdd {
project, _ := utils.ParseRepository(repoToAdd) project, _ := utils.ParseRepository(repoToAdd)
pullCount, err := dao.CountPull(repoToAdd) pullCount, err := dao.CountPull(repoToAdd)
@ -85,7 +85,7 @@ func SyncRegistry(pm promgr.ProjectManager) error {
if err := dao.AddRepository(repoRecord); err != nil { if err := dao.AddRepository(repoRecord); err != nil {
log.Errorf("Error happens when adding the missing repository: %v", err) log.Errorf("Error happens when adding the missing repository: %v", err)
} else { } else {
log.Debugf("Add repository: %s success.", repoToAdd) log.Infof("Add repository: %s success.", repoToAdd)
} }
} }
} }

4
tests/coverage4gotest.sh
View File

@ -21,13 +21,15 @@ function listDeps(){
} }
packages=$(go list ./... | grep -v -E 'vendor|tests|testing') packages=$(go list ./... | grep -v -E 'vendor|tests|testing')
echo testing packages: $packages
for package in $packages for package in $packages
do do
listDeps $package listDeps $package
# echo "DEBUG: testing package $package" # echo "DEBUG: testing package $package"
go test -race -cover -coverprofile=profile.tmp -coverpkg "$deps" $package echo go test -race -v -cover -coverprofile=profile.tmp -coverpkg "$deps" $package
go test -race -v -cover -coverprofile=profile.tmp -coverpkg "$deps" $package
if [ -f profile.tmp ] if [ -f profile.tmp ]
then then
cat profile.tmp | tail -n +2 >> profile.cov cat profile.tmp | tail -n +2 >> profile.cov

2
tests/docker-compose.test.yml
View File

@ -24,6 +24,6 @@ services:
image: goharbor/redis-photon:4.0 image: goharbor/redis-photon:4.0
restart: always restart: always
volumes: volumes:
- /data/redis:/data - /data/redis:/var/lib/redis
ports: ports:
- 6379:6379 - 6379:6379

4
tests/pushimage.sh
View File

@ -1,6 +1,8 @@
#!/bin/bash #!/bin/bash
set -x
set -e
IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'` IP=$(hostname -I | awk '{print $1}')
docker pull hello-world docker pull hello-world
docker pull busybox docker pull busybox
docker login -u admin -p Harbor12345 $IP:5000 docker login -u admin -p Harbor12345 $IP:5000

3
tests/travis/api_common_install.sh
View File

@ -4,6 +4,7 @@ set -x
set +e set +e
sudo rm -fr /data/* sudo rm -fr /data/*
sudo mkdir -p /data sudo mkdir -p /data
DIR="$(cd "$(dirname "$0")" && pwd)"
set -e set -e
if [ -z "$1" ]; then echo no ip specified; exit 1;fi if [ -z "$1" ]; then echo no ip specified; exit 1;fi
@ -20,7 +21,7 @@ fi
# prepare a chart file for API_DB test... # prepare a chart file for API_DB test...
sudo curl -o /home/travis/gopath/src/github.com/goharbor/harbor/tests/apitests/python/mariadb-4.3.1.tgz https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz sudo curl -o $DIR/../../tests/apitests/python/mariadb-4.3.1.tgz https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz
sudo apt-get update && sudo apt-get install -y --no-install-recommends python-dev openjdk-7-jdk libssl-dev && sudo apt-get autoremove -y && sudo rm -rf /var/lib/apt/lists/* sudo apt-get update && sudo apt-get install -y --no-install-recommends python-dev openjdk-7-jdk libssl-dev && sudo apt-get autoremove -y && sudo rm -rf /var/lib/apt/lists/*
sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.0.4 robotframework-httplibrary requests dbbot robotframework-pabot --upgrade sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.0.4 robotframework-httplibrary requests dbbot robotframework-pabot --upgrade

7
tests/travis/api_run.sh
View File

@ -17,6 +17,7 @@ harbor_logs_bucket="harbor-ci-logs"
#echo "[GSUtil]" >> $botofile #echo "[GSUtil]" >> $botofile
#echo "content_language = en" >> $botofile #echo "content_language = en" >> $botofile
#echo "default_project_id = $GS_PROJECT_ID" >> $botofile #echo "default_project_id = $GS_PROJECT_ID" >> $botofile
DIR="$(cd "$(dirname "$0")" && pwd)"
# GS util # GS util
function uploader { function uploader {
@ -30,16 +31,16 @@ docker ps
# run db auth api cases # run db auth api cases
if [ "$1" = 'DB' ]; then if [ "$1" = 'DB' ]; then
pybot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 /home/travis/gopath/src/github.com/goharbor/harbor/tests/robot-cases/Group0-BAT/API_DB.robot pybot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_DB.robot
elif [ "$1" = 'LDAP' ]; then elif [ "$1" = 'LDAP' ]; then
# run ldap api cases # run ldap api cases
python /home/travis/gopath/src/github.com/goharbor/harbor/tests/configharbor.py -H $IP -u $HARBOR_ADMIN -p $HARBOR_ADMIN_PASSWD -c auth_mode=ldap_auth \ python $DIR/../../tests/configharbor.py -H $IP -u $HARBOR_ADMIN -p $HARBOR_ADMIN_PASSWD -c auth_mode=ldap_auth \
ldap_url=ldap://$IP \ ldap_url=ldap://$IP \
ldap_search_dn=cn=admin,dc=example,dc=com \ ldap_search_dn=cn=admin,dc=example,dc=com \
ldap_search_password=admin \ ldap_search_password=admin \
ldap_base_dn=dc=example,dc=com \ ldap_base_dn=dc=example,dc=com \
ldap_uid=cn ldap_uid=cn
pybot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 /home/travis/gopath/src/github.com/goharbor/harbor/tests/robot-cases/Group0-BAT/API_LDAP.robot pybot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_LDAP.robot
else else
rc=999 rc=999
fi fi

2
tests/travis/ui_ut_run.sh
View File

@ -3,7 +3,7 @@ set -x
set -e set -e
cd ./src/portal cd ./src/portal
npm install -g -q --no-progress angular-cli npm install -g -q --no-progress @angular/cli
npm install -g -q --no-progress karma npm install -g -q --no-progress karma
npm install -q --no-progress npm install -q --no-progress
npm run test && cd - npm run test && cd -

10
tests/travis/ut_install.sh
View File

@ -14,9 +14,10 @@ go get github.com/stretchr/testify
go get golang.org/x/tools/cmd/cover go get golang.org/x/tools/cmd/cover
go get github.com/mattn/goveralls go get github.com/mattn/goveralls
go get -u github.com/client9/misspell/cmd/misspell go get -u github.com/client9/misspell/cmd/misspell
sudo service postgresql stop sudo service postgresql stop || echo no postgresql need to be stopped
sleep 2 sleep 2
sudo rm -rf /data/*
sudo -E env "PATH=$PATH" make go_check sudo -E env "PATH=$PATH" make go_check
sudo ./tests/hostcfg.sh sudo ./tests/hostcfg.sh
sudo ./tests/generateCerts.sh sudo ./tests/generateCerts.sh
@ -28,7 +29,10 @@ sudo mkdir -p /harbor && sudo mv ./VERSION /harbor/UIVERSION
sudo ./tests/testprepare.sh sudo ./tests/testprepare.sh
cd tests && sudo ./ldapprepare.sh && cd .. cd tests && sudo ./ldapprepare.sh && cd ..
sudo sed -i 's/__reg_version__/${REG_VERSION}-dev/g' ./make/docker-compose.test.yml env
docker images
sudo sed -i "s/__reg_version__/${REG_VERSION}-dev/g" ./make/docker-compose.test.yml
sudo sed -i 's/__version__/dev/g' ./make/docker-compose.test.yml sudo sed -i 's/__version__/dev/g' ./make/docker-compose.test.yml
cat ./make/docker-compose.test.yml
sudo mkdir -p ./make/common/config/registry/ && sudo mv ./tests/reg_config.yml ./make/common/config/registry/config.yml sudo mkdir -p ./make/common/config/registry/ && sudo mv ./tests/reg_config.yml ./make/common/config/registry/config.yml
sudo mkdir /storage && sudo chown 10000:10000 -R /storage sudo mkdir -p /storage && sudo chown 10000:10000 -R /storage

7
tests/travis/ut_run.sh
View File

@ -6,14 +6,15 @@ set -e
export POSTGRESQL_HOST=$1 export POSTGRESQL_HOST=$1
export REGISTRY_URL=$1:5000 export REGISTRY_URL=$1:5000
export CHROME_BIN=chromium-browser export CHROME_BIN=chromium-browser
export DISPLAY=:99.0 #export DISPLAY=:99.0
sh -e /etc/init.d/xvfb start #sh -e /etc/init.d/xvfb start
sudo docker-compose -f ./make/docker-compose.test.yml up -d sudo docker-compose -f ./make/docker-compose.test.yml up -d
sleep 10 sleep 10
./tests/pushimage.sh ./tests/pushimage.sh
docker ps docker ps
DIR="$(cd "$(dirname "$0")" && pwd)"
go test -race -i ./src/core ./src/jobservice go test -race -i ./src/core ./src/jobservice
sudo -E env "PATH=$PATH" "POSTGRES_MIGRATION_SCRIPTS_PATH=/home/travis/gopath/src/github.com/goharbor/harbor/make/migrations/postgresql/" ./tests/coverage4gotest.sh sudo -E env "PATH=$PATH" "POSTGRES_MIGRATION_SCRIPTS_PATH=$DIR/../../make/migrations/postgresql/" ./tests/coverage4gotest.sh
goveralls -coverprofile=profile.cov -service=travis-ci || true goveralls -coverprofile=profile.cov -service=travis-ci || true