Merge pull request #10280 from bitsf/cicd_github_action

enhance[cicd] introduce github action for CICD
2024-12-19 15:17:43 +01:00 · 2019-12-18 16:19:38 +08:00 · 2019-12-18 16:19:38 +08:00 · 9a1e77ca7c
commit 9a1e77ca7c
parent b89f81bde0 e32649adb4
13 changed files with 309 additions and 23 deletions
--- a/.github/workflows/CI.yml
+++ b/.github/workflows/CI.yml
@ -0,0 +1,277 @@
+name: CI
+env:
+   POSTGRESQL_HOST: localhost
+   POSTGRESQL_PORT: 5432
+   POSTGRESQL_USR: postgres
+   POSTGRESQL_PWD: root123
+   POSTGRESQL_DATABASE: registry
+   ADMINSERVER_URL: http://127.0.0.1:8888
+   DOCKER_COMPOSE_VERSION: 1.23.0
+   HARBOR_ADMIN: admin
+   HARBOR_ADMIN_PASSWD: Harbor12345
+   CORE_SECRET: tempString
+   KEY_PATH: "/data/secret/keys/secretkey"
+   REDIS_HOST: localhost
+   REG_VERSION: v2.7.1-patch-2819-2553
+   UI_BUILDER_VERSION: 1.6.0
+
+on: 
+  pull_request:
+  push:
+
+jobs:
+  UTTEST:
+    env:
+       UTTEST: true
+    runs-on: 
+      #- self-hosted
+      - ubuntu-latest
+    steps:
+      - name: Set up Go 1.13
+        uses: actions/setup-go@v1
+        with:
+           go-version: 1.13
+        id: go
+      - name: setup Docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+        with:
+          docker_version: 18.09
+          docker_channel: stable
+      - uses: actions/checkout@v1
+        with:
+         fetch-depth: 1
+         path: src/github.com/goharbor/harbor
+      - name: setup env
+        run: |
+          pwd
+          go env
+          echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
+          echo "::add-path::$(go env GOPATH)/bin"
+          echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
+        shell: bash
+      - name: before_install
+        run: |
+          set -x
+          pwd
+          env
+          #sudo apt install -y xvfb
+          #xvfb-run ls
+          curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+          chmod +x docker-compose
+          sudo mv docker-compose /usr/local/bin
+          IP=`hostname -I | awk '{print $1}'`
+          echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
+          echo "::set-env name=IP::$IP"
+          sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
+          sudo update-ca-certificates
+          sudo service docker restart
+      - name: install
+        run: |
+          env
+          bash ./tests/showtime.sh ./tests/travis/ut_install.sh
+      - name: script
+        run: |
+          echo IP: $IP
+          bash ./tests/showtime.sh ./tests/travis/ut_run.sh $IP
+        
+  APITEST_DB:
+    env:
+      APITEST_DB: true
+    runs-on:
+      #- self-hosted
+      - ubuntu-latest
+    steps:
+      - name: Set up Go 1.13
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+        id: go
+      - name: setup Docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+        with:
+          docker_version: 18.09
+          docker_channel: stable
+      - uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+          path: src/github.com/goharbor/harbor
+      - name: setup env
+        run: |
+          pwd
+          go env
+          echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
+          echo "::add-path::$(go env GOPATH)/bin"
+          echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
+        shell: bash
+      - name: before_install
+        run: |
+          set -x
+          pwd
+          env
+          #sudo apt install -y xvfb
+          #xvfb-run ls
+          curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+          chmod +x docker-compose
+          sudo mv docker-compose /usr/local/bin
+          IP=`hostname -I | awk '{print $1}'`
+          echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
+          echo "::set-env name=IP::$IP"
+          sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
+          sudo update-ca-certificates
+          sudo service docker restart
+      - name: install
+        run: |
+          env
+          bash ./tests/showtime.sh ./tests/travis/api_common_install.sh $IP DB
+      - name: script
+        run: |
+          echo IP: $IP
+          bash ./tests/showtime.sh ./tests/travis/api_run.sh DB $IP
+        
+  APITEST_LDAP:
+    env:
+      APITEST_LDAP: true
+    runs-on:
+      #- self-hosted
+      - ubuntu-latest
+    steps:
+      - name: Set up Go 1.13
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+        id: go
+      - name: setup Docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+        with:
+          docker_version: 18.09
+          docker_channel: stable
+      - uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+          path: src/github.com/goharbor/harbor
+      - name: setup env
+        run: |
+          pwd
+          go env
+          echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
+          echo "::add-path::$(go env GOPATH)/bin"
+          echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
+        shell: bash
+      - name: before_install
+        run: |
+          set -x
+          pwd
+          env
+          #sudo apt install -y xvfb
+          #xvfb-run ls
+          curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+          chmod +x docker-compose
+          sudo mv docker-compose /usr/local/bin
+          IP=`hostname -I | awk '{print $1}'`
+          echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
+          echo "::set-env name=IP::$IP"
+          sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
+          sudo update-ca-certificates
+          sudo service docker restart
+      - name: install
+        run: |
+          env
+          bash ./tests/showtime.sh ./tests/travis/api_common_install.sh $IP LDAP
+      - name: script
+        run: |
+          echo IP: $IP
+          bash ./tests/showtime.sh ./tests/travis/api_run.sh LDAP $IP
+
+  OFFLINE:
+    env:
+      OFFLINE: true
+    runs-on:
+      #- self-hosted
+      - ubuntu-latest
+    steps:
+      - name: Set up Go 1.13
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+        id: go
+      - name: setup Docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+        with:
+          docker_version: 18.09
+          docker_channel: stable
+      - uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+          path: src/github.com/goharbor/harbor
+      - name: setup env
+        run: |
+          pwd
+          docker version
+          go env
+          echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
+          echo "::add-path::$(go env GOPATH)/bin"
+          echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
+        shell: bash
+      - name: before_install
+        run: |
+          set -x
+          pwd
+          env
+          #sudo apt install -y xvfb
+          #xvfb-run ls
+          curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+          chmod +x docker-compose
+          sudo mv docker-compose /usr/local/bin
+          IP=`hostname -I | awk '{print $1}'`
+          echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
+          echo "::set-env name=IP::$IP"
+          sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
+          sudo update-ca-certificates
+          sudo service docker restart
+      - name: script
+        run: |
+          echo IP: $IP
+          bash ./tests/showtime.sh ./tests/travis/distro_installer.sh
+
+  UI_UT:
+    env:
+      UI_UT: true
+    runs-on:
+      #- self-hosted
+      - ubuntu-latest
+    steps:
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '10.16.2'
+      - uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+          path: src/github.com/goharbor/harbor
+      - name: setup env
+        run: |
+          pwd
+          go env
+          echo "::set-env name=GOPATH::$(go env GOPATH):$RUNNER_WORKSPACE"
+          echo "::add-path::$(go env GOPATH)/bin"
+          echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/tests/private_key.pem"
+        shell: bash
+      - name: before_install
+        run: |
+          set -x
+          pwd
+          env
+          #sudo apt install -y xvfb
+          #xvfb-run ls
+          #curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+          #chmod +x docker-compose
+          #sudo mv docker-compose /usr/local/bin
+          IP=`hostname -I | awk '{print $1}'`
+          #echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
+          echo "::set-env name=IP::$IP"
+          sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
+          sudo update-ca-certificates
+          #sudo service docker restart
+      - name: script
+        run: |
+          echo IP: $IP
+          bash ./tests/showtime.sh ./tests/travis/ui_ut_run.sh
--- a/make/common.sh
+++ b/make/common.sh
@ -89,13 +89,12 @@ function check_docker {
 		docker_version_part1=${BASH_REMATCH[2]}
 		docker_version_part2=${BASH_REMATCH[3]}

+		note "docker version: $docker_version"
 		# the version of docker does not meet the requirement
 		if [ "$docker_version_part1" -lt 17 ] || ([ "$docker_version_part1" -eq 17 ] && [ "$docker_version_part2" -lt 6 ])
 		then
 			error "Need to upgrade docker package to 17.06.0+."
 			exit 1
-		else
-			note "docker version: $docker_version"
 		fi
 	else
 		error "Failed to parse docker version."
@ -117,13 +116,12 @@ function check_dockercompose {
 		docker_compose_version_part1=${BASH_REMATCH[2]}
 		docker_compose_version_part2=${BASH_REMATCH[3]}

+		note "docker-compose version: $docker_compose_version"
 		# the version of docker-compose does not meet the requirement
 		if [ "$docker_compose_version_part1" -lt 1 ] || ([ "$docker_compose_version_part1" -eq 1 ] && [ "$docker_compose_version_part2" -lt 18 ])
 		then
 			error "Need to upgrade docker-compose package to 1.18.0+."
-                        exit 1
-		else
-			note "docker-compose version: $docker_compose_version"
+			exit 1
 		fi
 	else
 		error "Failed to parse docker-compose version."
--- a/make/photon/registry/Dockerfile
+++ b/make/photon/registry/Dockerfile
@ -16,5 +16,5 @@ USER harbor

 ENTRYPOINT ["/home/harbor/entrypoint.sh"]

-VOLUME ["/var/lib/registry"]
+VOLUME ["/storage"]
 EXPOSE 5000
--- a/src/core/api/api_test.go
+++ b/src/core/api/api_test.go
@ -174,7 +174,7 @@ func runCodeCheckingCases(t *testing.T, cases ...*codeCheckingCase) {
 			if resp.Body.Len() > 0 {
 				t.Log(resp.Body.String())
 			}
-			continue
+			t.FailNow()
 		}

 		if c.postFunc != nil {
--- a/src/core/api/utils.go
+++ b/src/core/api/utils.go
@ -64,7 +64,7 @@ func SyncRegistry(pm promgr.ProjectManager) error {
 	}

 	if len(reposToAdd) > 0 {
-		log.Debugf("Start adding repositories into DB... ")
+		log.Infof("Start adding repositories into DB %v ... ", len(reposToAdd))
 		for _, repoToAdd := range reposToAdd {
 			project, _ := utils.ParseRepository(repoToAdd)
 			pullCount, err := dao.CountPull(repoToAdd)
@ -85,7 +85,7 @@ func SyncRegistry(pm promgr.ProjectManager) error {
 			if err := dao.AddRepository(repoRecord); err != nil {
 				log.Errorf("Error happens when adding the missing repository: %v", err)
 			} else {
-				log.Debugf("Add repository: %s success.", repoToAdd)
+				log.Infof("Add repository: %s success.", repoToAdd)
 			}
 		}
 	}
--- a/tests/coverage4gotest.sh
+++ b/tests/coverage4gotest.sh
@ -21,13 +21,15 @@ function listDeps(){
 }

 packages=$(go list ./... | grep -v -E 'vendor|tests|testing')
+echo testing packages: $packages

 for package in $packages
 do
 	listDeps $package

 #    echo "DEBUG: testing package $package"
-	go test -race -cover -coverprofile=profile.tmp -coverpkg "$deps" $package
+	echo go test -race -v -cover -coverprofile=profile.tmp -coverpkg "$deps" $package
+	go test -race -v -cover -coverprofile=profile.tmp -coverpkg "$deps" $package
 	if [ -f profile.tmp ]	
 	then
 		cat profile.tmp | tail -n +2 >> profile.cov
--- a/tests/docker-compose.test.yml
+++ b/tests/docker-compose.test.yml
@ -24,6 +24,6 @@ services:
    image: goharbor/redis-photon:4.0
    restart: always
    volumes:
-      - /data/redis:/data
+      - /data/redis:/var/lib/redis
    ports:
      - 6379:6379
--- a/tests/pushimage.sh
+++ b/tests/pushimage.sh
@ -1,6 +1,8 @@
 #!/bin/bash
+set -x
+set -e

-IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'`
+IP=$(hostname -I | awk '{print $1}')
 docker pull hello-world
 docker pull busybox
 docker login -u admin -p Harbor12345 $IP:5000  
--- a/tests/travis/api_common_install.sh
+++ b/tests/travis/api_common_install.sh
@ -4,6 +4,7 @@ set -x
 set +e
 sudo rm -fr /data/*
 sudo mkdir -p /data
+DIR="$(cd "$(dirname "$0")" && pwd)"

 set -e
 if [ -z "$1" ]; then echo no ip specified; exit 1;fi
@ -20,7 +21,7 @@ fi


 # prepare a chart file for API_DB test...
-sudo curl -o /home/travis/gopath/src/github.com/goharbor/harbor/tests/apitests/python/mariadb-4.3.1.tgz https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz
+sudo curl -o $DIR/../../tests/apitests/python/mariadb-4.3.1.tgz https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz

 sudo apt-get update && sudo apt-get install -y --no-install-recommends python-dev openjdk-7-jdk libssl-dev && sudo apt-get autoremove -y && sudo rm -rf /var/lib/apt/lists/*
 sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.0.4 robotframework-httplibrary requests dbbot robotframework-pabot --upgrade
--- a/tests/travis/api_run.sh
+++ b/tests/travis/api_run.sh
@ -17,6 +17,7 @@ harbor_logs_bucket="harbor-ci-logs"
 #echo "[GSUtil]" >> $botofile
 #echo "content_language = en" >> $botofile
 #echo "default_project_id = $GS_PROJECT_ID" >> $botofile
+DIR="$(cd "$(dirname "$0")" && pwd)"

 # GS util
 function uploader {
@ -30,16 +31,16 @@ docker ps

 # run db auth api cases
 if [ "$1" = 'DB' ]; then
-    pybot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 /home/travis/gopath/src/github.com/goharbor/harbor/tests/robot-cases/Group0-BAT/API_DB.robot
+    pybot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_DB.robot
 elif [ "$1" = 'LDAP' ]; then
    # run ldap api cases
-    python /home/travis/gopath/src/github.com/goharbor/harbor/tests/configharbor.py -H $IP -u $HARBOR_ADMIN -p $HARBOR_ADMIN_PASSWD -c auth_mode=ldap_auth \
+    python $DIR/../../tests/configharbor.py -H $IP -u $HARBOR_ADMIN -p $HARBOR_ADMIN_PASSWD -c auth_mode=ldap_auth \
                                  ldap_url=ldap://$IP \
                                  ldap_search_dn=cn=admin,dc=example,dc=com \
                                  ldap_search_password=admin \
                                  ldap_base_dn=dc=example,dc=com \
                                  ldap_uid=cn
-    pybot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 /home/travis/gopath/src/github.com/goharbor/harbor/tests/robot-cases/Group0-BAT/API_LDAP.robot
+    pybot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_LDAP.robot
 else
    rc=999
 fi
--- a/tests/travis/ui_ut_run.sh
+++ b/tests/travis/ui_ut_run.sh
@ -3,7 +3,7 @@ set -x
 set -e

 cd ./src/portal
-npm install -g -q --no-progress angular-cli
+npm install -g -q --no-progress @angular/cli
 npm install -g -q --no-progress karma
 npm install -q --no-progress
 npm run test && cd -
--- a/tests/travis/ut_install.sh
+++ b/tests/travis/ut_install.sh
@ -14,9 +14,10 @@ go get github.com/stretchr/testify
 go get golang.org/x/tools/cmd/cover
 go get github.com/mattn/goveralls
 go get -u github.com/client9/misspell/cmd/misspell
-sudo service postgresql stop
+sudo service postgresql stop || echo no postgresql need to be stopped
 sleep 2

+sudo rm -rf /data/*
 sudo -E env "PATH=$PATH" make go_check
 sudo ./tests/hostcfg.sh
 sudo ./tests/generateCerts.sh
@ -28,7 +29,10 @@ sudo mkdir -p /harbor && sudo mv ./VERSION /harbor/UIVERSION
 sudo ./tests/testprepare.sh

 cd tests && sudo ./ldapprepare.sh && cd ..
-sudo sed -i 's/__reg_version__/${REG_VERSION}-dev/g' ./make/docker-compose.test.yml
+env
+docker images
+sudo sed -i "s/__reg_version__/${REG_VERSION}-dev/g" ./make/docker-compose.test.yml
 sudo sed -i 's/__version__/dev/g' ./make/docker-compose.test.yml
+cat ./make/docker-compose.test.yml
 sudo mkdir -p ./make/common/config/registry/ && sudo mv ./tests/reg_config.yml ./make/common/config/registry/config.yml
-sudo mkdir /storage && sudo chown 10000:10000 -R /storage
+sudo mkdir -p /storage && sudo chown 10000:10000 -R /storage
--- a/tests/travis/ut_run.sh
+++ b/tests/travis/ut_run.sh
@ -6,14 +6,15 @@ set -e
 export POSTGRESQL_HOST=$1
 export REGISTRY_URL=$1:5000
 export CHROME_BIN=chromium-browser
-export DISPLAY=:99.0
-sh -e /etc/init.d/xvfb start
+#export DISPLAY=:99.0
+#sh -e /etc/init.d/xvfb start

 sudo docker-compose -f ./make/docker-compose.test.yml up -d
 sleep 10
 ./tests/pushimage.sh
 docker ps

+DIR="$(cd "$(dirname "$0")" && pwd)"
 go test -race -i ./src/core ./src/jobservice
-sudo -E env "PATH=$PATH" "POSTGRES_MIGRATION_SCRIPTS_PATH=/home/travis/gopath/src/github.com/goharbor/harbor/make/migrations/postgresql/" ./tests/coverage4gotest.sh
+sudo -E env "PATH=$PATH" "POSTGRES_MIGRATION_SCRIPTS_PATH=$DIR/../../make/migrations/postgresql/" ./tests/coverage4gotest.sh
 goveralls -coverprofile=profile.cov -service=travis-ci || true