Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-24 01:27:49 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #10280 from bitsf/cicd_github_action

Browse Source 
enhance[cicd] introduce github action for CICD
This commit is contained in:
Ziming 2019-12-18 16:19:38 +08:00 committed by GitHub
commit 9a1e77ca7c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 309 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

277
.github/workflows/CI.yml vendored Normal file
View File

@ -0,0 +1,277 @@
name: CI
env:
POSTGRESQL_HOST: localhost
POSTGRESQL_PORT: 5432
POSTGRESQL_USR: postgres
POSTGRESQL_PWD: root123
POSTGRESQL_DATABASE: registry
ADMINSERVER_URL: http://127.0.0.1:8888
DOCKER_COMPOSE_VERSION: 1.23.0
HARBOR_ADMIN: admin
HARBOR_ADMIN_PASSWD: Harbor12345
CORE_SECRET: tempString
KEY_PATH: "/data/secret/keys/secretkey"
REDIS_HOST: localhost
REG_VERSION: v2.7.1-patch-2819-2553
UI_BUILDER_VERSION: 1.6.0
on:
pull_request:
push:
jobs:
UTTEST:
env:
UTTEST: true
runs-on:
#- self-hosted
- ubuntu-latest
steps:
- name: Set up Go 1.13
uses: actions/setup-go@v1
with:
go-version: 1.13
id: go
- name: setup Docker
uses: docker-practice/actions-setup-docker@0.0.1
with:
docker_version: 18.09
docker_channel: stable
- uses: actions/checkout@v1
with:
fetch-depth: 1
path: src/github.com/goharbor/harbor
- name: setup env
run: |
pwd
go env
echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
echo "::add-path::$(go env GOPATH)/bin"
echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
shell: bash
- name: before_install
run: |
set -x
pwd
env
#sudo apt install -y xvfb
#xvfb-run ls
curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
chmod +x docker-compose
sudo mv docker-compose /usr/local/bin
IP=`hostname -I | awk '{print $1}'`
echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
echo "::set-env name=IP::$IP"
sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
sudo update-ca-certificates
sudo service docker restart
- name: install
run: |
env
bash ./tests/showtime.sh ./tests/travis/ut_install.sh
- name: script
run: |
echo IP: $IP
bash ./tests/showtime.sh ./tests/travis/ut_run.sh $IP
APITEST_DB:
env:
APITEST_DB: true
runs-on:
#- self-hosted
- ubuntu-latest
steps:
- name: Set up Go 1.13
uses: actions/setup-go@v1
with:
go-version: 1.13
id: go
- name: setup Docker
uses: docker-practice/actions-setup-docker@0.0.1
with:
docker_version: 18.09
docker_channel: stable
- uses: actions/checkout@v1
with:
fetch-depth: 1
path: src/github.com/goharbor/harbor
- name: setup env
run: |
pwd
go env
echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
echo "::add-path::$(go env GOPATH)/bin"
echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
shell: bash
- name: before_install
run: |
set -x
pwd
env
#sudo apt install -y xvfb
#xvfb-run ls
curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
chmod +x docker-compose
sudo mv docker-compose /usr/local/bin
IP=`hostname -I | awk '{print $1}'`
echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
echo "::set-env name=IP::$IP"
sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
sudo update-ca-certificates
sudo service docker restart
- name: install
run: |
env
bash ./tests/showtime.sh ./tests/travis/api_common_install.sh $IP DB
- name: script
run: |
echo IP: $IP
bash ./tests/showtime.sh ./tests/travis/api_run.sh DB $IP
APITEST_LDAP:
env:
APITEST_LDAP: true
runs-on:
#- self-hosted
- ubuntu-latest
steps:
- name: Set up Go 1.13
uses: actions/setup-go@v1
with:
go-version: 1.13
id: go
- name: setup Docker
uses: docker-practice/actions-setup-docker@0.0.1
with:
docker_version: 18.09
docker_channel: stable
- uses: actions/checkout@v1
with:
fetch-depth: 1
path: src/github.com/goharbor/harbor
- name: setup env
run: |
pwd
go env
echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
echo "::add-path::$(go env GOPATH)/bin"
echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
shell: bash
- name: before_install
run: |
set -x
pwd
env
#sudo apt install -y xvfb
#xvfb-run ls
curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
chmod +x docker-compose
sudo mv docker-compose /usr/local/bin
IP=`hostname -I | awk '{print $1}'`
echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
echo "::set-env name=IP::$IP"
sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
sudo update-ca-certificates
sudo service docker restart
- name: install
run: |
env
bash ./tests/showtime.sh ./tests/travis/api_common_install.sh $IP LDAP
- name: script
run: |
echo IP: $IP
bash ./tests/showtime.sh ./tests/travis/api_run.sh LDAP $IP
OFFLINE:
env:
OFFLINE: true
runs-on:
#- self-hosted
- ubuntu-latest
steps:
- name: Set up Go 1.13
uses: actions/setup-go@v1
with:
go-version: 1.13
id: go
- name: setup Docker
uses: docker-practice/actions-setup-docker@0.0.1
with:
docker_version: 18.09
docker_channel: stable
- uses: actions/checkout@v1
with:
fetch-depth: 1
path: src/github.com/goharbor/harbor
- name: setup env
run: |
pwd
docker version
go env
echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
echo "::add-path::$(go env GOPATH)/bin"
echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
shell: bash
- name: before_install
run: |
set -x
pwd
env
#sudo apt install -y xvfb
#xvfb-run ls
curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
chmod +x docker-compose
sudo mv docker-compose /usr/local/bin
IP=`hostname -I | awk '{print $1}'`
echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
echo "::set-env name=IP::$IP"
sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
sudo update-ca-certificates
sudo service docker restart
- name: script
run: |
echo IP: $IP
bash ./tests/showtime.sh ./tests/travis/distro_installer.sh
UI_UT:
env:
UI_UT: true
runs-on:
#- self-hosted
- ubuntu-latest
steps:
- uses: actions/setup-node@v1
with:
node-version: '10.16.2'
- uses: actions/checkout@v1
with:
fetch-depth: 1
path: src/github.com/goharbor/harbor
- name: setup env
run: |
pwd
go env
echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
echo "::add-path::$(go env GOPATH)/bin"
echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
shell: bash
- name: before_install
run: |
set -x
pwd
env
#sudo apt install -y xvfb
#xvfb-run ls
#curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
#chmod +x docker-compose
#sudo mv docker-compose /usr/local/bin
IP=`hostname -I | awk '{print $1}'`
#echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
echo "::set-env name=IP::$IP"
sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
sudo update-ca-certificates
#sudo service docker restart
- name: script
run: |
echo IP: $IP
bash ./tests/showtime.sh ./tests/travis/ui_ut_run.sh

8
make/common.sh
View File

@ -89,13 +89,12 @@ function check_docker {
docker_version_part1=${BASH_REMATCH[2]} docker_version_part1=${BASH_REMATCH[2]}
docker_version_part2=${BASH_REMATCH[3]} docker_version_part2=${BASH_REMATCH[3]}
note "docker version: $docker_version"
# the version of docker does not meet the requirement # the version of docker does not meet the requirement
if [ "$docker_version_part1" -lt 17 ] || ([ "$docker_version_part1" -eq 17 ] && [ "$docker_version_part2" -lt 6 ]) if [ "$docker_version_part1" -lt 17 ] || ([ "$docker_version_part1" -eq 17 ] && [ "$docker_version_part2" -lt 6 ])
then then
error "Need to upgrade docker package to 17.06.0+." error "Need to upgrade docker package to 17.06.0+."
exit 1 exit 1
else
note "docker version: $docker_version"
fi fi
else else
error "Failed to parse docker version." error "Failed to parse docker version."
@ -117,13 +116,12 @@ function check_dockercompose {
docker_compose_version_part1=${BASH_REMATCH[2]} docker_compose_version_part1=${BASH_REMATCH[2]}
docker_compose_version_part2=${BASH_REMATCH[3]} docker_compose_version_part2=${BASH_REMATCH[3]}
note "docker-compose version: $docker_compose_version"
# the version of docker-compose does not meet the requirement # the version of docker-compose does not meet the requirement
if [ "$docker_compose_version_part1" -lt 1 ] || ([ "$docker_compose_version_part1" -eq 1 ] && [ "$docker_compose_version_part2" -lt 18 ]) if [ "$docker_compose_version_part1" -lt 1 ] || ([ "$docker_compose_version_part1" -eq 1 ] && [ "$docker_compose_version_part2" -lt 18 ])
then then
error "Need to upgrade docker-compose package to 1.18.0+." error "Need to upgrade docker-compose package to 1.18.0+."
exit 1 exit 1
else
note "docker-compose version: $docker_compose_version"
fi fi
else else
error "Failed to parse docker-compose version." error "Failed to parse docker-compose version."

2
make/photon/registry/Dockerfile
View File

@ -16,5 +16,5 @@ USER harbor
ENTRYPOINT ["/home/harbor/entrypoint.sh"] ENTRYPOINT ["/home/harbor/entrypoint.sh"]
VOLUME ["/var/lib/registry"] VOLUME ["/storage"]
EXPOSE 5000 EXPOSE 5000

2
src/core/api/api_test.go
View File

@ -174,7 +174,7 @@ func runCodeCheckingCases(t *testing.T, cases ...*codeCheckingCase) {
if resp.Body.Len() > 0 { if resp.Body.Len() > 0 {
t.Log(resp.Body.String()) t.Log(resp.Body.String())
} }
continue t.FailNow()
} }
if c.postFunc != nil { if c.postFunc != nil {

4
src/core/api/utils.go
View File

@ -64,7 +64,7 @@ func SyncRegistry(pm promgr.ProjectManager) error {
} }
if len(reposToAdd) > 0 { if len(reposToAdd) > 0 {
log.Debugf("Start adding repositories into DB... ") log.Infof("Start adding repositories into DB %v ... ", len(reposToAdd))
for _, repoToAdd := range reposToAdd { for _, repoToAdd := range reposToAdd {
project, _ := utils.ParseRepository(repoToAdd) project, _ := utils.ParseRepository(repoToAdd)
pullCount, err := dao.CountPull(repoToAdd) pullCount, err := dao.CountPull(repoToAdd)
@ -85,7 +85,7 @@ func SyncRegistry(pm promgr.ProjectManager) error {
if err := dao.AddRepository(repoRecord); err != nil { if err := dao.AddRepository(repoRecord); err != nil {
log.Errorf("Error happens when adding the missing repository: %v", err) log.Errorf("Error happens when adding the missing repository: %v", err)
} else { } else {
log.Debugf("Add repository: %s success.", repoToAdd) log.Infof("Add repository: %s success.", repoToAdd)
} }
} }
} }

4
tests/coverage4gotest.sh
View File

@ -21,13 +21,15 @@ function listDeps(){
} }
packages=$(go list ./... | grep -v -E 'vendor|tests|testing') packages=$(go list ./... | grep -v -E 'vendor|tests|testing')
echo testing packages: $packages
for package in $packages for package in $packages
do do
listDeps $package listDeps $package
# echo "DEBUG: testing package $package" # echo "DEBUG: testing package $package"
go test -race -cover -coverprofile=profile.tmp -coverpkg "$deps" $package echo go test -race -v -cover -coverprofile=profile.tmp -coverpkg "$deps" $package
go test -race -v -cover -coverprofile=profile.tmp -coverpkg "$deps" $package
if [ -f profile.tmp ] if [ -f profile.tmp ]
then then
cat profile.tmp | tail -n +2 >> profile.cov cat profile.tmp | tail -n +2 >> profile.cov

2
tests/docker-compose.test.yml
View File

@ -24,6 +24,6 @@ services:
image: goharbor/redis-photon:4.0 image: goharbor/redis-photon:4.0
restart: always restart: always
volumes: volumes:
- /data/redis:/data - /data/redis:/var/lib/redis
ports: ports:
- 6379:6379 - 6379:6379

4
tests/pushimage.sh
View File

@ -1,6 +1,8 @@
#!/bin/bash #!/bin/bash
set -x
set -e
IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'` IP=$(hostname -I | awk '{print $1}')
docker pull hello-world docker pull hello-world
docker pull busybox docker pull busybox
docker login -u admin -p Harbor12345 $IP:5000 docker login -u admin -p Harbor12345 $IP:5000

3
tests/travis/api_common_install.sh
View File

@ -4,6 +4,7 @@ set -x
set +e set +e
sudo rm -fr /data/* sudo rm -fr /data/*
sudo mkdir -p /data sudo mkdir -p /data
DIR="$(cd "$(dirname "$0")" && pwd)"
set -e set -e
if [ -z "$1" ]; then echo no ip specified; exit 1;fi if [ -z "$1" ]; then echo no ip specified; exit 1;fi
@ -20,7 +21,7 @@ fi
# prepare a chart file for API_DB test... # prepare a chart file for API_DB test...
sudo curl -o /home/travis/gopath/src/github.com/goharbor/harbor/tests/apitests/python/mariadb-4.3.1.tgz https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz sudo curl -o $DIR/../../tests/apitests/python/mariadb-4.3.1.tgz https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz
sudo apt-get update && sudo apt-get install -y --no-install-recommends python-dev openjdk-7-jdk libssl-dev && sudo apt-get autoremove -y && sudo rm -rf /var/lib/apt/lists/* sudo apt-get update && sudo apt-get install -y --no-install-recommends python-dev openjdk-7-jdk libssl-dev && sudo apt-get autoremove -y && sudo rm -rf /var/lib/apt/lists/*
sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.0.4 robotframework-httplibrary requests dbbot robotframework-pabot --upgrade sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.0.4 robotframework-httplibrary requests dbbot robotframework-pabot --upgrade

7
tests/travis/api_run.sh
View File

@ -17,6 +17,7 @@ harbor_logs_bucket="harbor-ci-logs"
#echo "[GSUtil]" >> $botofile #echo "[GSUtil]" >> $botofile
#echo "content_language = en" >> $botofile #echo "content_language = en" >> $botofile
#echo "default_project_id = $GS_PROJECT_ID" >> $botofile #echo "default_project_id = $GS_PROJECT_ID" >> $botofile
DIR="$(cd "$(dirname "$0")" && pwd)"
# GS util # GS util
function uploader { function uploader {
@ -30,16 +31,16 @@ docker ps
# run db auth api cases # run db auth api cases
if [ "$1" = 'DB' ]; then if [ "$1" = 'DB' ]; then
pybot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 /home/travis/gopath/src/github.com/goharbor/harbor/tests/robot-cases/Group0-BAT/API_DB.robot pybot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_DB.robot
elif [ "$1" = 'LDAP' ]; then elif [ "$1" = 'LDAP' ]; then
# run ldap api cases # run ldap api cases
python /home/travis/gopath/src/github.com/goharbor/harbor/tests/configharbor.py -H $IP -u $HARBOR_ADMIN -p $HARBOR_ADMIN_PASSWD -c auth_mode=ldap_auth \ python $DIR/../../tests/configharbor.py -H $IP -u $HARBOR_ADMIN -p $HARBOR_ADMIN_PASSWD -c auth_mode=ldap_auth \
ldap_url=ldap://$IP \ ldap_url=ldap://$IP \
ldap_search_dn=cn=admin,dc=example,dc=com \ ldap_search_dn=cn=admin,dc=example,dc=com \
ldap_search_password=admin \ ldap_search_password=admin \
ldap_base_dn=dc=example,dc=com \ ldap_base_dn=dc=example,dc=com \
ldap_uid=cn ldap_uid=cn
pybot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 /home/travis/gopath/src/github.com/goharbor/harbor/tests/robot-cases/Group0-BAT/API_LDAP.robot pybot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_LDAP.robot
else else
rc=999 rc=999
fi fi

2
tests/travis/ui_ut_run.sh
View File

@ -3,7 +3,7 @@ set -x
set -e set -e
cd ./src/portal cd ./src/portal
npm install -g -q --no-progress angular-cli npm install -g -q --no-progress @angular/cli
npm install -g -q --no-progress karma npm install -g -q --no-progress karma
npm install -q --no-progress npm install -q --no-progress
npm run test && cd - npm run test && cd -

10
tests/travis/ut_install.sh
View File

@ -14,9 +14,10 @@ go get github.com/stretchr/testify
go get golang.org/x/tools/cmd/cover go get golang.org/x/tools/cmd/cover
go get github.com/mattn/goveralls go get github.com/mattn/goveralls
go get -u github.com/client9/misspell/cmd/misspell go get -u github.com/client9/misspell/cmd/misspell
sudo service postgresql stop sudo service postgresql stop || echo no postgresql need to be stopped
sleep 2 sleep 2
sudo rm -rf /data/*
sudo -E env "PATH=$PATH" make go_check sudo -E env "PATH=$PATH" make go_check
sudo ./tests/hostcfg.sh sudo ./tests/hostcfg.sh
sudo ./tests/generateCerts.sh sudo ./tests/generateCerts.sh
@ -28,7 +29,10 @@ sudo mkdir -p /harbor && sudo mv ./VERSION /harbor/UIVERSION
sudo ./tests/testprepare.sh sudo ./tests/testprepare.sh
cd tests && sudo ./ldapprepare.sh && cd .. cd tests && sudo ./ldapprepare.sh && cd ..
sudo sed -i 's/__reg_version__/${REG_VERSION}-dev/g' ./make/docker-compose.test.yml env
docker images
sudo sed -i "s/__reg_version__/${REG_VERSION}-dev/g" ./make/docker-compose.test.yml
sudo sed -i 's/__version__/dev/g' ./make/docker-compose.test.yml sudo sed -i 's/__version__/dev/g' ./make/docker-compose.test.yml
cat ./make/docker-compose.test.yml
sudo mkdir -p ./make/common/config/registry/ && sudo mv ./tests/reg_config.yml ./make/common/config/registry/config.yml sudo mkdir -p ./make/common/config/registry/ && sudo mv ./tests/reg_config.yml ./make/common/config/registry/config.yml
sudo mkdir /storage && sudo chown 10000:10000 -R /storage sudo mkdir -p /storage && sudo chown 10000:10000 -R /storage

7
tests/travis/ut_run.sh
View File

@ -6,14 +6,15 @@ set -e
export POSTGRESQL_HOST=$1 export POSTGRESQL_HOST=$1
export REGISTRY_URL=$1:5000 export REGISTRY_URL=$1:5000
export CHROME_BIN=chromium-browser export CHROME_BIN=chromium-browser
export DISPLAY=:99.0 #export DISPLAY=:99.0
sh -e /etc/init.d/xvfb start #sh -e /etc/init.d/xvfb start
sudo docker-compose -f ./make/docker-compose.test.yml up -d sudo docker-compose -f ./make/docker-compose.test.yml up -d
sleep 10 sleep 10
./tests/pushimage.sh ./tests/pushimage.sh
docker ps docker ps
DIR="$(cd "$(dirname "$0")" && pwd)"
go test -race -i ./src/core ./src/jobservice go test -race -i ./src/core ./src/jobservice
sudo -E env "PATH=$PATH" "POSTGRES_MIGRATION_SCRIPTS_PATH=/home/travis/gopath/src/github.com/goharbor/harbor/make/migrations/postgresql/" ./tests/coverage4gotest.sh sudo -E env "PATH=$PATH" "POSTGRES_MIGRATION_SCRIPTS_PATH=$DIR/../../make/migrations/postgresql/" ./tests/coverage4gotest.sh
goveralls -coverprofile=profile.cov -service=travis-ci || true goveralls -coverprofile=profile.cov -service=travis-ci || true