From 9a2188641dfa050da2680919163d6d05911bd738 Mon Sep 17 00:00:00 2001 From: danfengliu Date: Tue, 11 Aug 2020 02:34:58 +0000 Subject: [PATCH] Upgrade python and robot for drone build 1.Separate Dockerfile to drone and nightly e2e, one Dockerfile should cover both scenarios, it will be resolved later; 2.Fix docker api err type issue, API error has no message attibute; 3.Upgrade docker in nightly e2e images. Signed-off-by: danfengliu --- .drone.yml | 2 +- .github/workflows/CI.yml | 3 - tests/apitests/python/library/docker_api.py | 12 +-- tests/ci/api_common_install.sh | 4 + tests/e2e-image/drone/Dockerfile | 94 +++++++++++++++++++ tests/e2e-image/{ => nightly}/Dockerfile | 6 +- tests/generateCerts.sh | 6 +- tests/integration.sh | 2 +- .../package-offline-installer.robot | 34 +------ .../package-online-installer.robot | 32 +------ 10 files changed, 116 insertions(+), 79 deletions(-) create mode 100644 tests/e2e-image/drone/Dockerfile rename tests/e2e-image/{ => nightly}/Dockerfile (94%) diff --git a/.drone.yml b/.drone.yml index b98b00bda..cfa6bc8b9 100644 --- a/.drone.yml +++ b/.drone.yml @@ -27,7 +27,7 @@ pipeline: status: success test-and-issue-build: - image: goharbor/harbor-e2e-engine:1.43 + image: goharbor/harbor-drone-engine:2.0 pull: true privileged: true environment: diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 96b2cd0e8..5270404ac 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -233,9 +233,6 @@ jobs: IP=`hostname -I | awk '{print $1}'` echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json echo "::set-env name=IP::$IP" - sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/ - sudo update-ca-certificates - sudo service docker restart - name: install run: | cd src/github.com/goharbor/harbor diff --git a/tests/apitests/python/library/docker_api.py b/tests/apitests/python/library/docker_api.py index c01fa59c2..b745b8ef9 100644 --- a/tests/apitests/python/library/docker_api.py +++ b/tests/apitests/python/library/docker_api.py @@ -91,7 +91,7 @@ class DockerAPI(object): if str(err).lower().find(expected_error_message.lower()) < 0: raise Exception(r"Docker login: Return message {} is not as expected {}".format(str(err), expected_error_message)) else: - raise Exception(r" Docker login failed, error is [{}]".format (err.message)) + raise Exception(r" Docker login failed, error is [{}]".format (str(err))) def docker_image_pull(self, image, tag = None, expected_error_message = None): if tag is not None: @@ -112,7 +112,7 @@ class DockerAPI(object): if str(err).lower().find(expected_error_message.lower()) < 0: raise Exception(r"Pull image: Return message {} is not as expected {}".format(str(err), expected_error_message)) else: - raise Exception(r" Docker pull image {} failed, error is [{}]".format (image, err.message)) + raise Exception(r" Docker pull image {} failed, error is [{}]".format (image, message)) if caught_err == False: if expected_error_message is not None: if str(ret).lower().find(expected_error_message.lower()) < 0: @@ -128,8 +128,8 @@ class DockerAPI(object): try: self.DCLIENT.tag(image, harbor_registry, _tag, force=True) return harbor_registry, _tag - except docker.errors.APIError as e: - raise Exception(r" Docker tag image {} failed, error is [{}]".format (image, e.message)) + except docker.errors.APIError as err: + raise Exception(r" Docker tag image {} failed, error is [{}]".format (image, str(err))) def docker_image_push(self, harbor_registry, tag, expected_error_message = None): caught_err = False @@ -146,7 +146,7 @@ class DockerAPI(object): if str(err).lower().find(expected_error_message.lower()) < 0: raise Exception(r"Push image: Return message {} is not as expected {}".format(str(err), expected_error_message)) else: - raise Exception(r" Docker push image {} failed, error is [{}]".format (harbor_registry, err.message)) + raise Exception(r" Docker push image {} failed, error is [{}]".format (harbor_registry, message)) if caught_err == False: if expected_error_message is not None: if str(ret).lower().find(expected_error_message.lower()) < 0: @@ -191,7 +191,7 @@ class DockerAPI(object): if str(err).lower().find(expected_error_message.lower()) < 0: raise Exception(r"Push image: Return message {} is not as expected {}".format(str(err), expected_error_message)) else: - raise Exception(r" Docker build image {} failed, error is [{}]".format (harbor_registry, err.message)) + raise Exception(r" Docker build image {} failed, error is [{}]".format (harbor_registry, str(err))) if caught_err == False: if expected_error_message is not None: if str(ret).lower().find(expected_error_message.lower()) < 0: diff --git a/tests/ci/api_common_install.sh b/tests/ci/api_common_install.sh index 63be2ed02..c3e8dedce 100755 --- a/tests/ci/api_common_install.sh +++ b/tests/ci/api_common_install.sh @@ -12,6 +12,10 @@ if [ -z "$1" ]; then echo no ip specified; exit 1;fi sudo ./tests/generateCerts.sh $1 sudo mkdir -p /etc/docker/certs.d/$1 && sudo cp ./tests/harbor_ca.crt /etc/docker/certs.d/$1/ && rm -rf ~/.docker/ && mkdir -p ~/.docker/tls/$1:4443/ && sudo cp ./tests/harbor_ca.crt ~/.docker/tls/$1:4443/ +sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/ +sudo update-ca-certificates +sudo service docker restart + sudo ./tests/hostcfg.sh if [ "$2" = 'LDAP' ]; then diff --git a/tests/e2e-image/drone/Dockerfile b/tests/e2e-image/drone/Dockerfile new file mode 100644 index 000000000..cf58b65a0 --- /dev/null +++ b/tests/e2e-image/drone/Dockerfile @@ -0,0 +1,94 @@ +FROM ubuntu:18.04 +ENV LANG C.UTF-8 + + +RUN apt-get update && apt-get install -y --no-install-recommends \ + jq \ + bc \ + time \ + gcc \ + python-dev \ + libffi-dev \ + libssl-dev \ + sshpass \ + ant \ + ant-optional \ + xvfb \ + libxi6 \ + libgconf-2-4 \ + openjdk-8-jdk \ + rpcbind \ + nfs-common \ + unzip \ + zip \ + bzip2 \ + wget curl \ + parted \ + #ip tool + ethtool \ + iproute2 \ + bridge-utils \ + iputils-ping \ + # Add docker in docker support + btrfs-tools \ + e2fsprogs \ + iptables \ + xfsprogs \ + dnsutils \ + netcat \ + # Speed up ISO builds with already installed reqs + yum \ + yum-utils \ + cpio \ + rpm \ + ca-certificates \ + xz-utils \ + xorriso \ + sendmail && \ + # Cleanup + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +RUN apt-get update -y && \ + apt-get install -y zbar-tools libzbar-dev python-zbar +RUN dpkg -L libzbar-dev; ls -l /usr/include/zbar.h +RUN apt-get update -y +RUN apt-get install -y python3.6 +RUN rm /usr/bin/python +RUN ln -s /usr/bin/python3.6 /usr/bin/python +RUN apt-get install -y python3-pip + + + +RUN apt-get update && apt install libnss3-tools && \ + mkdir -p $HOME/.pki/nssdb && \ + echo Harbor12345 > password.ca && \ + certutil -d sql:$HOME/.pki/nssdb -N -f password.ca + +RUN pip3 install pyasn1 google-apitools==0.5.31 gsutil robotframework==3.2.1 robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-seleniumlibrary==4.3.0 robotframework-pabot robotframework-JSONLibrary --upgrade + +# Install docker, docker compose +RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-18.06.3-ce.tgz && \ + tar --strip-components=1 -xvzf docker-18.06.3-ce.tgz -C /usr/bin && \ + curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose && \ + chmod +x /usr/local/bin/docker-compose + +RUN wget https://github.com/drone/drone-cli/releases/download/v0.8.3/drone_linux_amd64.tar.gz && tar zxf drone_linux_amd64.tar.gz && \ + install -t /usr/local/bin drone && mv drone drone_src + +RUN set -x \ + && groupadd --system dockremap \ + && adduser --system --ingroup dockremap dockremap \ + && echo 'dockremap:165536:65536' >> /etc/subuid \ + && echo 'dockremap:165536:65536' >> /etc/subgid + +ENV DIND_COMMIT 3b5fac462d21ca164b3778647420016315289034 + +RUN wget "https://raw.githubusercontent.com/docker/docker/${DIND_COMMIT}/hack/dind" -O /usr/local/bin/dind \ + && chmod +x /usr/local/bin/dind + +# This container needs to be run in privileged mode(run with --privileged option) to make it work +COPY dockerd-entrypoint.sh /usr/local/bin/dockerd-entrypoint.sh +RUN chmod +x /usr/local/bin/dockerd-entrypoint.sh + +VOLUME /var/lib/docker \ No newline at end of file diff --git a/tests/e2e-image/Dockerfile b/tests/e2e-image/nightly/Dockerfile similarity index 94% rename from tests/e2e-image/Dockerfile rename to tests/e2e-image/nightly/Dockerfile index 75a6bb8b6..82908b07a 100644 --- a/tests/e2e-image/Dockerfile +++ b/tests/e2e-image/nightly/Dockerfile @@ -1,5 +1,7 @@ FROM golang:1.11.2 +#docker build -f drone/Dockerfile -t goharbor/harbor-e2e-engine:2.4 . +# V2.4.1: Upgrade docker engine RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - RUN sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' @@ -75,8 +77,8 @@ RUN wget https://bootstrap.pypa.io/get-pip.py && \ pip install pyasn1 google-apitools==0.5.15 gsutil robotframework==3.2.1 robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-seleniumlibrary==4.3.0 robotframework-pabot robotframework-JSONLibrary --upgrade # Install docker, docker compose -RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-18.06.3-ce.tgz && \ - tar --strip-components=1 -xvzf docker-18.06.3-ce.tgz -C /usr/bin && \ +RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-19.03.12.tgz && \ + tar --strip-components=1 -xvzf docker-19.03.12.tgz -C /usr/bin && \ curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose && \ chmod +x /usr/local/bin/docker-compose diff --git a/tests/generateCerts.sh b/tests/generateCerts.sh index 2f95789ed..023e46650 100755 --- a/tests/generateCerts.sh +++ b/tests/generateCerts.sh @@ -18,9 +18,9 @@ if [[ -z ${OPENSSLCNF} ]]; then fi # Create CA certificate -#openssl req \ -# -newkey rsa:4096 -nodes -sha256 -keyout $CUR_DIR/harbor_ca.key \ -# -x509 -days 365 -out $CUR_DIR/harbor_ca.crt -subj '/C=CN/ST=PEK/L=Bei Jing/O=VMware/CN=HarborCA' +openssl req \ + -newkey rsa:4096 -nodes -sha256 -keyout $CUR_DIR/harbor_ca.key \ + -x509 -days 365 -out $CUR_DIR/harbor_ca.crt -subj '/C=CN/ST=PEK/L=Bei Jing/O=VMware/CN=HarborCA' # Generate a Certificate Signing Request if echo $IP|grep -E '^([0-9]+\.){3}[0-9]+$' ; then diff --git a/tests/integration.sh b/tests/integration.sh index 0e7126000..dd2c8622e 100755 --- a/tests/integration.sh +++ b/tests/integration.sh @@ -115,7 +115,7 @@ function uploader { function package_installer { echo "Package Harbor offline installer." - pybot --removekeywords TAG:secret --include Bundle tests/robot-cases/Group0-Distro-Harbor + robot --removekeywords TAG:secret --include Bundle tests/robot-cases/Group0-Distro-Harbor harbor_offline_build_bundle=$(basename harbor-offline-installer-*.tgz) harbor_online_build_bundle=$(basename harbor-online-installer-*.tgz) upload_build=true diff --git a/tests/robot-cases/Group0-Distro-Harbor/package-offline-installer.robot b/tests/robot-cases/Group0-Distro-Harbor/package-offline-installer.robot index b2ea064d4..1f53ea6e2 100644 --- a/tests/robot-cases/Group0-Distro-Harbor/package-offline-installer.robot +++ b/tests/robot-cases/Group0-Distro-Harbor/package-offline-installer.robot @@ -14,39 +14,9 @@ *** Settings *** Documentation Harbor BATs -Library Selenium2Library -Library OperatingSystem -Library Process +Resource ../../resources/Util.robot Default Tags Bundle -*** Keywords *** -Start Docker Daemon Locally - ${pid}= Run pidof dockerd - #${rc} ${output}= Run And Return Rc And Output ./tests/robot-cases/Group0-Util/docker_config.sh - #Log ${output} - #Should Be Equal As Integers ${rc} 0 - Return From Keyword If '${pid}' != '${EMPTY}' - OperatingSystem.File Should Exist /usr/local/bin/dockerd-entrypoint.sh - ${handle}= Start Process /usr/local/bin/dockerd-entrypoint.sh dockerd>./daemon-local.log 2>&1 shell=True - Process Should Be Running ${handle} - :FOR ${IDX} IN RANGE 5 - \ ${pid}= Run pidof dockerd - \ Exit For Loop If '${pid}' != '${EMPTY}' - \ Sleep 2s - Sleep 2s - [Return] ${handle} - -Package Harbor Offline - [Arguments] ${with_notary}=true ${with_clair}=true ${with_chartmuseum}=true ${with_trivy}=true - Log To Console \nStart Docker Daemon - Start Docker Daemon Locally - Log To Console \nMake Offline Package - Log To Console \n\nmake package_offline GOBUILDTAGS="include_oss include_gcs" BASEIMAGETAG=%{Harbor_Build_Base_Tag} NPM_REGISTRY=%{NPM_REGISTRY} VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum} TRIVYFLAG=${with_trivy} HTTPPROXY= - ${rc} ${output}= Run And Return Rc And Output make package_offline GOBUILDTAGS="include_oss include_gcs" BASEIMAGETAG=%{Harbor_Build_Base_Tag} NPM_REGISTRY=%{NPM_REGISTRY} VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum} TRIVYFLAG=${with_trivy} HTTPPROXY= - Log To Console ${rc} - Log To Console ${output} - Should Be Equal As Integers ${rc} 0 - *** Test Cases *** Distro Harbor Offline - Package Harbor Offline + Package Harbor Offline \ No newline at end of file diff --git a/tests/robot-cases/Group0-Distro-Harbor/package-online-installer.robot b/tests/robot-cases/Group0-Distro-Harbor/package-online-installer.robot index fd9902146..48de57634 100644 --- a/tests/robot-cases/Group0-Distro-Harbor/package-online-installer.robot +++ b/tests/robot-cases/Group0-Distro-Harbor/package-online-installer.robot @@ -14,39 +14,9 @@ *** Settings *** Documentation Harbor BATs -Library Selenium2Library -Library OperatingSystem -Library Process +Resource ../../resources/Util.robot Default Tags Bundle -*** Keywords *** -Start Docker Daemon Locally - ${pid}= Run pidof dockerd - #${rc} ${output}= Run And Return Rc And Output ./tests/robot-cases/Group0-Util/docker_config.sh - #Log ${output} - #Should Be Equal As Integers ${rc} 0 - Return From Keyword If '${pid}' != '${EMPTY}' - OperatingSystem.File Should Exist /usr/local/bin/dockerd-entrypoint.sh - ${handle}= Start Process /usr/local/bin/dockerd-entrypoint.sh dockerd>./daemon-local.log 2>&1 shell=True - Process Should Be Running ${handle} - :FOR ${IDX} IN RANGE 5 - \ ${pid}= Run pidof dockerd - \ Exit For Loop If '${pid}' != '${EMPTY}' - \ Sleep 2s - Sleep 2s - [Return] ${handle} - -Package Harbor Online - [Arguments] ${with_notary}=true ${with_clair}=true ${with_chartmuseum}=true ${with_trivy}=true - Log To Console \nStart Docker Daemon - Start Docker Daemon Locally - Log To Console \nMake Online Package - Log To Console \nmake package_online GOBUILDTAGS="include_oss include_gcs" VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum} TRIVYFLAG=${with_trivy} HTTPPROXY= - ${rc} ${output}= Run And Return Rc And Output make package_online GOBUILDTAGS="include_oss include_gcs" VERSIONTAG=%{Harbor_Assets_Version} PKGVERSIONTAG=%{Harbor_Package_Version} NOTARYFLAG=${with_notary} CLAIRFLAG=${with_clair} CHARTFLAG=${with_chartmuseum} TRIVYFLAG=${with_trivy} HTTPPROXY= - Log ${rc} - Log ${output} - Should Be Equal As Integers ${rc} 0 - *** Test Cases *** Distro Harbor Online Package Harbor Online \ No newline at end of file