Merge pull request #4018 from wy65701436/refine-build-process

[Upload Build] Refine CI build process and storage structure
This commit is contained in:
Daniel Jiang 2018-01-17 10:21:14 +08:00 committed by GitHub
commit 7195865226
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 73 additions and 98 deletions

View File

@ -10,7 +10,7 @@ pipeline:
tags: true
recursive: false
integration-test-on-pr:
test-and-issue-build:
image: vmware/harbor-e2e-engine:1.38
pull: true
privileged: true
@ -36,27 +36,6 @@ pipeline:
when:
status: success
bundle:
image: vmware/harbor-e2e-engine:1.38
pull: true
privileged: true
environment:
BIN: bin
GOPATH: /drone
SHELL: /bin/bash
BUILD_NUMBER: ${DRONE_BUILD_NUMBER}
commands:
- du -ks harbor-offline-installer-*.tgz | awk '{print $1 / 1024}' | { read x; echo $x MB; }
- mkdir -p bundle
- cp harbor-offline-installer-*.tgz bundle
- cp harbor-offline-installer-*.tgz bundle/harbor-offline-installer-latest.tgz
- ls -la bundle
when:
repo: vmware/harbor
event: [ push, tag ]
branch: [ master, release-*, pks-*, refs/tags/* ]
status: success
notify-slack:
image: plugins/slack
webhook: ${SLACK_URL}
@ -68,34 +47,6 @@ pipeline:
branch: [ master, release-*, refs/tags/* ]
status: [ failure, success ]
publish-gcs-builds:
image: maplain/drone-gcs:latest
pull: true
source: bundle
target: harbor-builds
acl:
- allUsers:READER
cache_control: public,max-age=3600
when:
repo: vmware/harbor
event: [ push, tag ]
branch: [ master ]
status: success
publish-gcs-releases:
image: maplain/drone-gcs:latest
pull: true
source: bundle
target: harbor-releases
acl:
- allUsers:READER
cache_control: public,max-age=3600
when:
repo: vmware/harbor
event: [ push, tag ]
branch: [ release-*, refs/tags/* ]
status: success
trigger:
image: plugins/downstream
server: https://ci.vcna.io

View File

@ -1 +1 @@
eyJhbGciOiJIUzI1NiJ9.IyBIYXJib3IgZHJvbmUuCi0tLQp3b3Jrc3BhY2U6CiAgYmFzZTogL2Ryb25lCiAgcGF0aDogc3JjL2dpdGh1Yi5jb20vdm13YXJlL2hhcmJvcgoKcGlwZWxpbmU6CiAgY2xvbmU6CiAgICBpbWFnZTogcGx1Z2lucy9naXQKICAgIHRhZ3M6IHRydWUKICAgIHJlY3Vyc2l2ZTogZmFsc2UKCiAgaW50ZWdyYXRpb24tdGVzdC1vbi1wcjoKICAgIGltYWdlOiB2bXdhcmUvaGFyYm9yLWUyZS1lbmdpbmU6MS4zOAogICAgcHVsbDogdHJ1ZQogICAgcHJpdmlsZWdlZDogdHJ1ZQogICAgZW52aXJvbm1lbnQ6CiAgICAgIEJJTjogYmluCiAgICAgIEdPUEFUSDogL2Ryb25lCiAgICAgIFNIRUxMOiAvYmluL2Jhc2gKICAgICAgTE9HX1RFTVBfRElSOiBpbnN0YWxsLWxvZ3MKICAgICAgR0lUSFVCX0FVVE9NQVRJT05fQVBJX0tFWTogICR7R0lUSFVCX0FVVE9NQVRJT05fQVBJX0tFWX0KICAgICAgRFJPTkVfU0VSVkVSOiAgJHtEUk9ORV9TRVJWRVJ9CiAgICAgIERST05FX1RPS0VOOiAgJHtEUk9ORV9UT0tFTl9JTlRFfQogICAgICBIQVJCT1JfQURNSU46ICR7SEFSQk9SX0FETUlOfQogICAgICBIQVJCT1JfUEFTU1dPUkQ6ICR7SEFSQk9SX1BBU1NXT1JEfQogICAgICBHU19QUk9KRUNUX0lEOiAke0dTX1BST0pFQ1RfSUR9CiAgICAgIEdTX0NMSUVOVF9FTUFJTDogJHtHU19DTElFTlRfRU1BSUx9CiAgICAgIEdTX1BSSVZBVEVfS0VZOiAke0dTX1BSSVZBVEVfS0VZfQogICAgICBET01BSU46ICR7Q0lfRE9NQUlOfQogICAgICBNQUlMX1BXRDogJHtNQUlMX1BXRH0KICAgICAgTlBNX1VTRVJOQU1FOiAke05QTV9VU0VSTkFNRX0KICAgICAgTlBNX1BBU1NXT1JEOiAke05QTV9QQVNTV09SRH0KICAgIGNvbW1hbmRzOgogICAgICAtIHRlc3RzL2ludGVncmF0aW9uLnNoCiAgICB3aGVuOgogICAgICBzdGF0dXM6IHN1Y2Nlc3MKCiAgYnVuZGxlOgogICAgaW1hZ2U6IHZtd2FyZS9oYXJib3ItZTJlLWVuZ2luZToxLjM4CiAgICBwdWxsOiB0cnVlCiAgICBwcml2aWxlZ2VkOiB0cnVlCiAgICBlbnZpcm9ubWVudDoKICAgICAgQklOOiBiaW4KICAgICAgR09QQVRIOiAvZHJvbmUKICAgICAgU0hFTEw6IC9iaW4vYmFzaAogICAgICBCVUlMRF9OVU1CRVI6ICR7RFJPTkVfQlVJTERfTlVNQkVSfQogICAgY29tbWFuZHM6CiAgICAgIC0gZHUgLWtzIGhhcmJvci1vZmZsaW5lLWluc3RhbGxlci0qLnRneiB8IGF3ayAne3ByaW50ICQxIC8gMTAyNH0nIHwgeyByZWFkIHg7IGVjaG8gJHggTUI7IH0KICAgICAgLSBta2RpciAtcCBidW5kbGUKICAgICAgLSBjcCBoYXJib3Itb2ZmbGluZS1pbnN0YWxsZXItKi50Z3ogYnVuZGxlCiAgICAgIC0gY3AgaGFyYm9yLW9mZmxpbmUtaW5zdGFsbGVyLSoudGd6IGJ1bmRsZS9oYXJib3Itb2ZmbGluZS1pbnN0YWxsZXItbGF0ZXN0LnRnegogICAgICAtIGxzIC1sYSBidW5kbGUKICAgIHdoZW46CiAgICAgIHJlcG86IHZtd2FyZS9oYXJib3IKICAgICAgZXZlbnQ6IFsgcHVzaCwgdGFnIF0KICAgICAgYnJhbmNoOiBbIG1hc3RlciwgcmVsZWFzZS0qLCBwa3MtKiwgcmVmcy90YWdzLyogXQogICAgICBzdGF0dXM6IHN1Y2Nlc3MKCiAgbm90aWZ5LXNsYWNrOgogICAgaW1hZ2U6IHBsdWdpbnMvc2xhY2sKICAgIHdlYmhvb2s6ICR7U0xBQ0tfVVJMfQogICAgdXNlcm5hbWU6IGRyb25lCiAgICB0ZW1wbGF0ZTogPgogICAgICBidWlsZCBodHRwczovL2NpLnZjbmEuaW8vdm13YXJlL2hhcmJvci97eyBidWlsZC5udW1iZXIgfX0gZmluaXNoZWQgd2l0aCBhIHt7IGJ1aWxkLnN0YXR1cyB9fSBzdGF0dXMuIFBsZWFzZSBmaW5kIGxvZ3MgYXQgaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2hhcmJvci1jaS1sb2dzL2ludGVncmF0aW9uX2xvZ3Nfe3sgYnVpbGQubnVtYmVyIH19X3t7IGJ1aWxkLmNvbW1pdCB9fS50YXIuZ3oKICAgIHdoZW46CiAgICAgIHJlcG86IHZtd2FyZS9oYXJib3IKICAgICAgYnJhbmNoOiBbIG1hc3RlciwgcmVsZWFzZS0qLCByZWZzL3RhZ3MvKiBdCiAgICAgIHN0YXR1czogWyBmYWlsdXJlLCBzdWNjZXNzIF0KCiAgcHVibGlzaC1nY3MtYnVpbGRzOgogICAgaW1hZ2U6IG1hcGxhaW4vZHJvbmUtZ2NzOmxhdGVzdAogICAgcHVsbDogdHJ1ZQogICAgc291cmNlOiBidW5kbGUKICAgIHRhcmdldDogaGFyYm9yLWJ1aWxkcwogICAgYWNsOgogICAgICAtIGFsbFVzZXJzOlJFQURFUgogICAgY2FjaGVfY29udHJvbDogcHVibGljLG1heC1hZ2U9MzYwMAogICAgd2hlbjoKICAgICAgcmVwbzogdm13YXJlL2hhcmJvcgogICAgICBldmVudDogWyBwdXNoLCB0YWcgXQogICAgICBicmFuY2g6IFsgbWFzdGVyIF0KICAgICAgc3RhdHVzOiBzdWNjZXNzCgogIHB1Ymxpc2gtZ2NzLXJlbGVhc2VzOgogICAgaW1hZ2U6IG1hcGxhaW4vZHJvbmUtZ2NzOmxhdGVzdAogICAgcHVsbDogdHJ1ZQogICAgc291cmNlOiBidW5kbGUKICAgIHRhcmdldDogaGFyYm9yLXJlbGVhc2VzCiAgICBhY2w6CiAgICAgIC0gYWxsVXNlcnM6UkVBREVSCiAgICBjYWNoZV9jb250cm9sOiBwdWJsaWMsbWF4LWFnZT0zNjAwCiAgICB3aGVuOgogICAgICByZXBvOiB2bXdhcmUvaGFyYm9yCiAgICAgIGV2ZW50OiBbIHB1c2gsIHRhZyBdCiAgICAgIGJyYW5jaDogWyByZWxlYXNlLSosIHJlZnMvdGFncy8qIF0KICAgICAgc3RhdHVzOiBzdWNjZXNzCgogIHRyaWdnZXI6CiAgICBpbWFnZTogcGx1Z2lucy9kb3duc3RyZWFtCiAgICBzZXJ2ZXI6IGh0dHBzOi8vY2kudmNuYS5pbwogICAgdG9rZW46ICR7RE9XTlNUUkVBTV9UT0tFTn0KICAgIGZvcms6IHRydWUKICAgIHJlcG9zaXRvcmllczoKICAgICAgIC0gdm13YXJlL3ZpYy1wcm9kdWN0CiAgICB3aGVuOgogICAgICByZXBvOiB2bXdhcmUvaGFyYm9yCiAgICAgIGV2ZW50OiBbIHB1c2gsIHRhZyBdCiAgICAgIGJyYW5jaDogWyBtYXN0ZXIsIHJlbGVhc2UtKiwgcmVmcy90YWdzLyogXQogICAgICBzdGF0dXM6IHN1Y2Nlc3MK.0TeBeHyYbP8xrqxi1RUDjXnB0tqChcuuDhNv4hbbUJs
eyJhbGciOiJIUzI1NiJ9.IyBIYXJib3IgZHJvbmUuCi0tLQp3b3Jrc3BhY2U6CiAgYmFzZTogL2Ryb25lCiAgcGF0aDogc3JjL2dpdGh1Yi5jb20vdm13YXJlL2hhcmJvcgoKcGlwZWxpbmU6CiAgY2xvbmU6CiAgICBpbWFnZTogcGx1Z2lucy9naXQKICAgIHRhZ3M6IHRydWUKICAgIHJlY3Vyc2l2ZTogZmFsc2UKCiAgdGVzdC1hbmQtaXNzdWUtYnVpbGQ6CiAgICBpbWFnZTogdm13YXJlL2hhcmJvci1lMmUtZW5naW5lOjEuMzgKICAgIHB1bGw6IHRydWUKICAgIHByaXZpbGVnZWQ6IHRydWUKICAgIGVudmlyb25tZW50OgogICAgICBCSU46IGJpbgogICAgICBHT1BBVEg6IC9kcm9uZQogICAgICBTSEVMTDogL2Jpbi9iYXNoCiAgICAgIExPR19URU1QX0RJUjogaW5zdGFsbC1sb2dzCiAgICAgIEdJVEhVQl9BVVRPTUFUSU9OX0FQSV9LRVk6ICAke0dJVEhVQl9BVVRPTUFUSU9OX0FQSV9LRVl9CiAgICAgIERST05FX1NFUlZFUjogICR7RFJPTkVfU0VSVkVSfQogICAgICBEUk9ORV9UT0tFTjogICR7RFJPTkVfVE9LRU5fSU5URX0KICAgICAgSEFSQk9SX0FETUlOOiAke0hBUkJPUl9BRE1JTn0KICAgICAgSEFSQk9SX1BBU1NXT1JEOiAke0hBUkJPUl9QQVNTV09SRH0KICAgICAgR1NfUFJPSkVDVF9JRDogJHtHU19QUk9KRUNUX0lEfQogICAgICBHU19DTElFTlRfRU1BSUw6ICR7R1NfQ0xJRU5UX0VNQUlMfQogICAgICBHU19QUklWQVRFX0tFWTogJHtHU19QUklWQVRFX0tFWX0KICAgICAgRE9NQUlOOiAke0NJX0RPTUFJTn0KICAgICAgTUFJTF9QV0Q6ICR7TUFJTF9QV0R9CiAgICAgIE5QTV9VU0VSTkFNRTogJHtOUE1fVVNFUk5BTUV9CiAgICAgIE5QTV9QQVNTV09SRDogJHtOUE1fUEFTU1dPUkR9CiAgICBjb21tYW5kczoKICAgICAgLSB0ZXN0cy9pbnRlZ3JhdGlvbi5zaAogICAgd2hlbjoKICAgICAgc3RhdHVzOiBzdWNjZXNzCgogIG5vdGlmeS1zbGFjazoKICAgIGltYWdlOiBwbHVnaW5zL3NsYWNrCiAgICB3ZWJob29rOiAke1NMQUNLX1VSTH0KICAgIHVzZXJuYW1lOiBkcm9uZQogICAgdGVtcGxhdGU6ID4KICAgICAgYnVpbGQgaHR0cHM6Ly9jaS52Y25hLmlvL3Ztd2FyZS9oYXJib3Ive3sgYnVpbGQubnVtYmVyIH19IGZpbmlzaGVkIHdpdGggYSB7eyBidWlsZC5zdGF0dXMgfX0gc3RhdHVzLiBQbGVhc2UgZmluZCBsb2dzIGF0IGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9oYXJib3ItY2ktbG9ncy9pbnRlZ3JhdGlvbl9sb2dzX3t7IGJ1aWxkLm51bWJlciB9fV97eyBidWlsZC5jb21taXQgfX0udGFyLmd6CiAgICB3aGVuOgogICAgICByZXBvOiB2bXdhcmUvaGFyYm9yCiAgICAgIGJyYW5jaDogWyBtYXN0ZXIsIHJlbGVhc2UtKiwgcmVmcy90YWdzLyogXQogICAgICBzdGF0dXM6IFsgZmFpbHVyZSwgc3VjY2VzcyBdCgogIHRyaWdnZXI6CiAgICBpbWFnZTogcGx1Z2lucy9kb3duc3RyZWFtCiAgICBzZXJ2ZXI6IGh0dHBzOi8vY2kudmNuYS5pbwogICAgdG9rZW46ICR7RE9XTlNUUkVBTV9UT0tFTn0KICAgIGZvcms6IHRydWUKICAgIHJlcG9zaXRvcmllczoKICAgICAgIC0gdm13YXJlL3ZpYy1wcm9kdWN0CiAgICB3aGVuOgogICAgICByZXBvOiB2bXdhcmUvaGFyYm9yCiAgICAgIGV2ZW50OiBbIHB1c2gsIHRhZyBdCiAgICAgIGJyYW5jaDogWyBtYXN0ZXIsIHJlbGVhc2UtKiwgcmVmcy90YWdzLyogXQogICAgICBzdGF0dXM6IHN1Y2Nlc3MK.uqtrh64v_8xYxgxnH8O7ynxySrO6Wgs7CKCDoVdeyP4

View File

@ -30,13 +30,21 @@ echo $buildinfo
upload_build=false
nightly_run=false
upload_latest_build=false
upload_bundle_success=false
latest_build_file='latest.build'
publish_npm=true
harbor_build_bundle=""
harbor_logs_bucket="harbor-ci-logs"
harbor_builds_bucket="harbor-builds"
harbor_releases_bucket="harbor-releases"
harbor_ci_pipeline_store_bucket="harbor-ci-pipeline-store/latest"
harbor_target_bucket=""
if [ $DRONE_BRANCH == "master" ]; then
harbor_target_bucket=$harbor_builds_bucket
else
harbor_target_bucket=$harbor_releases_bucket/$DRONE_BRANCH
fi
# GC credentials
keyfile="/root/harbor-ci-logs.key"
@ -54,24 +62,45 @@ echo $container_ip
# GS util
function uploader {
gsutil cp $1 gs://$2
gsutil -D setacl public-read gs://$2/$1 &> /dev/null
gsutil cp $1 gs://$2/$1
gsutil -D setacl public-read gs://$2/$1 &> /dev/null
}
function package_offline_installer {
echo "Package Harbor offline installer."
pybot --removekeywords TAG:secret --include Bundle tests/robot-cases/Group0-Distro-Harbor
harbor_build_bundle=$(basename harbor-offline-installer-*.tgz)
upload_build=true
echo "Package name is: $harbor_build_bundle"
du -ks $harbor_build_bundle | awk '{print $1 / 1024}' | { read x; echo $x MB; }
}
## --------------------------------------------- Run Test Case ---------------------------------------------
if [ $DRONE_REPO != "vmware/harbor" ]; then
echo "Only run tests again Harbor Repo."
exit 1
echo "Only run tests again Harbor Repo."
exit 1
fi
if [[ $DRONE_BRANCH == "master" || $DRONE_BRANCH == *"refs/tags"* || $DRONE_BRANCH == "release-"* ]] && [[ $DRONE_BUILD_EVENT == "push" || $DRONE_BUILD_EVENT == "tag" ]]; then
## -------------- Package installer with clean code -----------------
echo "Package Harbor build."
pybot --removekeywords TAG:secret --include Bundle tests/robot-cases/Group0-Distro-Harbor
echo "Running full CI for $DRONE_BUILD_EVENT on $DRONE_BRANCH"
upload_latest_build=true
pybot -v ip:$container_ip --removekeywords TAG:secret --include BAT tests/robot-cases/Group0-BAT
elif (echo $buildinfo | grep -q "\[Specific CI="); then
echo "--------------------------------------------------"
echo "Running CI for $DRONE_BUILD_EVENT on $DRONE_BRANCH"
echo "--------------------------------------------------"
##
# Any merge code or tag on branch master, release-* or pks-* will trigger package offline installer.
#
# Put code here is because that it needs clean code to build installer.
##
if [ $DRONE_BRANCH == "master" ] || [ $DRONE_BRANCH == *"refs/tags"* ] || [ $DRONE_BRANCH == "release-"* ] || [ $DRONE_BRANCH == "pks-"* ]; then
if [ $DRONE_BUILD_EVENT == "push" ] || [ $DRONE_BUILD_EVENT == "tag" ]; then
package_offline_installer
upload_latest_build=true
fi
fi
##
# Any Event(PR or merge code) on any branch will trigger test run.
##
if (echo $buildinfo | grep -q "\[Specific CI="); then
buildtype=$(echo $buildinfo | grep "\[Specific CI=")
testsuite=$(echo $buildtype | awk -F"\[Specific CI=" '{sub(/\].*/,"",$2);print $2}')
pybot -v ip:$container_ip --removekeywords TAG:secret --suite $testsuite tests/robot-cases
@ -80,17 +109,14 @@ elif (echo $buildinfo | grep -q "\[Full CI\]"); then
elif (echo $buildinfo | grep -q "\[Skip CI\]"); then
echo "Skip CI."
elif (echo $buildinfo | grep -q "\[Upload Build\]"); then
upload_latest_build=true
upload_build=true
echo "Package Harbor build."
pybot --removekeywords TAG:secret --include Bundle tests/robot-cases/Group0-Distro-Harbor
echo "Running full CI for $DRONE_BUILD_EVENT on $DRONE_BRANCH"
package_offline_installer
pybot -v ip:$container_ip --removekeywords TAG:secret --include BAT tests/robot-cases/Group0-BAT
else
# default mode is BAT.
pybot -v ip:$container_ip --removekeywords TAG:secret --include BAT tests/robot-cases/Group0-BAT
fi
# rc is used to identify test run pass or fail.
rc="$?"
echo $rc
@ -99,43 +125,41 @@ timestamp=$(date +%s)
outfile="integration_logs_"$DRONE_BUILD_NUMBER"_"$DRONE_COMMIT".tar.gz"
tar -zcvf $outfile output.xml log.html *.png package.list *container-logs.zip *.log /var/log/harbor/* /data/config/* /data/job_logs/*
if [ -f "$outfile" ]; then
uploader $outfile $harbor_logs_bucket
echo "----------------------------------------------"
echo "Download test logs:"
echo "https://storage.googleapis.com/harbor-ci-logs/$outfile"
echo "----------------------------------------------"
uploader $outfile $harbor_logs_bucket
echo "----------------------------------------------"
echo "Download test logs:"
echo "https://storage.googleapis.com/harbor-ci-logs/$outfile"
echo "----------------------------------------------"
else
echo "No log output file to upload"
echo "No log output file to upload"
fi
## --------------------------------------------- Upload Harbor Bundle File ---------------------------------------
if [ $upload_build == true ] && [ $rc -eq 0 ]; then
harbor_build_bundle=$(basename harbor-offline-installer-*.tgz)
uploader $harbor_build_bundle $harbor_builds_bucket
#
# Build storage structure:
#
# 1(master), harbor-builds/harbor-offline-installer-*.tgz
# latest.build
# harbor-offline-installer-latest.tgz
if [ $DRONE_BRANCH == "master" ]; then
cp $harbor_build_bundle harbor-offline-installer-latest-master.tgz
uploader harbor-offline-installer-latest-master.tgz $harbor_ci_pipeline_store_bucket
fi
if [[ $DRONE_BRANCH == *"refs/tags"* || $DRONE_BRANCH == "release-"* ]]; then
cp $harbor_build_bundle harbor-offline-installer-latest-release.tgz
uploader harbor-offline-installer-latest-release.tgz $harbor_ci_pipeline_store_bucket
fi
# 2(others), harbor-releases/${branch}/harbor-offline-installer-*.tgz
# latest.build
# harbor-offline-installer-latest.tgz
#
if [ $upload_build == true ] && [ $rc -eq 0 ]; then
cp $harbor_build_bundle harbor-offline-installer-latest.tgz
uploader $harbor_build_bundle $harbor_target_bucket
uploader harbor-offline-installer-latest.tgz $harbor_target_bucket
upload_bundle_success=true
fi
## --------------------------------------------- Upload Harbor Latest Build File ---------------------------------
if [ $upload_latest_build == true ] && [ $rc -eq 0 ]; then
echo "update latest build file."
harbor_build_bundle=$(basename harbor-offline-installer-*.tgz)
echo $harbor_build_bundle
if [[ $DRONE_BRANCH == "master" ]] && [[ $DRONE_BUILD_EVENT == "push" || $DRONE_BUILD_EVENT == "tag" ]]; then
echo 'https://storage.googleapis.com/harbor-builds/'$harbor_build_bundle > $latest_build_file
uploader $latest_build_file $harbor_builds_bucket
fi
if [[ $DRONE_BRANCH == *"refs/tags"* || $DRONE_BRANCH == "release-"* ]] && [[ $DRONE_BUILD_EVENT == "push" || $DRONE_BUILD_EVENT == "tag" ]]; then
echo 'https://storage.googleapis.com/harbor-releases/'$harbor_build_bundle > $latest_build_file
uploader $latest_build_file $harbor_releases_bucket
fi
## --------------------------------------------- Upload Harbor Latest Build File ----------------------------------
#
# latest.build file holds the latest offline installer url, it must be sure that the installer has been uploaded successfull.
#
if [ $upload_latest_build == true ] && [ $upload_bundle_success == true ] && [ $rc -eq 0 ]; then
echo 'https://storage.googleapis.com/'$harbor_target_bucket/$harbor_build_bundle > $latest_build_file
uploader $latest_build_file $harbor_target_bucket
fi
## ------------------------------------- Build & Publish NPM Package for VIC ------------------------------------