diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 72a30e1ef..14c57a829 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -113,7 +113,6 @@ jobs: cd src/github.com/goharbor/harbor pwd go env - echo "::set-env name=CNAB_PATH::$(go env GOPATH)/src/github.com/docker" echo "::set-env name=GITHUB_TOKEN::${{ secrets.GITHUB_TOKEN }}" echo "::set-env name=GOPATH::$(go env GOPATH):$GITHUB_WORKSPACE" echo "::add-path::$(go env GOPATH)/bin" @@ -133,40 +132,6 @@ jobs: curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose chmod +x docker-compose sudo mv docker-compose /usr/local/bin - echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json - sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/ - sudo update-ca-certificates - sudo service docker restart - wget https://get.helm.sh/helm-v2.14.1-linux-386.tar.gz && tar zxvf helm-v2.14.1-linux-386.tar.gz - sudo mv linux-386/helm /usr/local/bin/helm2 - helm2 init --client-only - helm2 plugin list | grep push || helm2 plugin install https://github.com/chartmuseum/helm-push - wget https://get.helm.sh/helm-v3.1.1-linux-386.tar.gz && tar zxvf helm-v3.1.1-linux-386.tar.gz - sudo mv linux-386/helm /usr/local/bin/helm3 - helm3 plugin list | grep push || helm3 plugin install https://github.com/chartmuseum/helm-push - rm -rf $CNAB_PATH;mkdir -p $CNAB_PATH && cd $CNAB_PATH && git clone https://github.com/cnabio/cnab-to-oci.git - cd cnab-to-oci && git checkout v0.3.0-beta4 - go list - make build - sudo mv bin/cnab-to-oci /usr/local/bin - curl -LO https://github.com/deislabs/oras/releases/download/v0.8.1/oras_0.8.1_linux_amd64.tar.gz - mkdir -p oras-install/ - tar -zxf oras_0.8.1_*.tar.gz -C oras-install/ - sudo mv oras-install/oras /usr/local/bin/ - sudo apt-get update && sudo apt-get install -y \ - build-essential \ - uuid-dev \ - libgpgme-dev \ - squashfs-tools \ - libseccomp-dev \ - pkg-config \ - cryptsetup-bin - export VERSION=3.5.3 && \ - wget https://github.com/sylabs/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz && \ - tar -xzf singularity-${VERSION}.tar.gz && cd singularity - ./mconfig && make -C builddir && sudo make -C builddir install - sudo apt-get update -y ; sudo apt-get install -y zbar-tools libzbar-dev python-zbar python3.7 - sudo rm /usr/bin/python ; sudo ln -s /usr/bin/python3.7 /usr/bin/python ; sudo apt-get install -y python3-pip - name: install run: | cd src/github.com/goharbor/harbor @@ -211,6 +176,8 @@ jobs: echo "::set-env name=GOPATH::$(go env GOPATH):$GITHUB_WORKSPACE" echo "::add-path::$(go env GOPATH)/bin" echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/src/github.com/goharbor/harbor/tests/private_key.pem" + IP=`hostname -I | awk '{print $1}'` + echo "::set-env name=IP::$IP" shell: bash - name: before_install run: | @@ -224,12 +191,6 @@ jobs: curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose chmod +x docker-compose sudo mv docker-compose /usr/local/bin - IP=`hostname -I | awk '{print $1}'` - echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json - echo "::set-env name=IP::$IP" - python -V - sudo apt-get update -y ; sudo apt-get install -y zbar-tools libzbar-dev python-zbar python3.7 - sudo rm /usr/bin/python ; sudo ln -s /usr/bin/python3.7 /usr/bin/python ; sudo apt-get install -y python3-pip - name: install run: | cd src/github.com/goharbor/harbor diff --git a/tests/apitests/python/library/singularity.py b/tests/apitests/python/library/singularity.py index b5bc9fd5b..2258b29b7 100644 --- a/tests/apitests/python/library/singularity.py +++ b/tests/apitests/python/library/singularity.py @@ -15,7 +15,7 @@ def singularity_push_to_harbor(harbor_server, sif_file, project, image, tag): ret = base.run_command( [singularity_cmd, "push", sif_file, "oras://"+harbor_server + "/" + project + "/" + image+":"+ tag] ) def singularity_pull(out_file, from_sif_file): - ret = base.run_command( [singularity_cmd, "pull", out_file, from_sif_file] ) + ret = base.run_command( [singularity_cmd, "pull", "--allow-unsigned", out_file, from_sif_file] ) def push_singularity_to_harbor(from_URI, from_namespace, harbor_server, user, password, project, image, tag): tmp_sif_file = image+timestamp+".sif" diff --git a/tests/apitests/python/test_list_helm_charts.py b/tests/apitests/python/test_list_helm_charts.py index 5b65226d5..3ca842bf4 100644 --- a/tests/apitests/python/test_list_helm_charts.py +++ b/tests/apitests/python/test_list_helm_charts.py @@ -4,6 +4,7 @@ import unittest from testutils import ADMIN_CLIENT, CHART_API_CLIENT from testutils import TEARDOWN +import base from library.user import User from library.project import Project from library.chart import Chart @@ -50,6 +51,7 @@ class TestProjects(unittest.TestCase): TestProjects.CHART_NAME = 'mariadb' TestProjects.VERSION = '4.3.1' + base.run_command( ["curl", r"-o", "./tests/apitests/python/mariadb-4.3.1.tgz", "https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz"]) #1. Create a new user(UA); TestProjects.user_chart_id, user_chart_name = self.user.create_user(user_password = user_chart_password, **ADMIN_CLIENT) diff --git a/tests/apitests/python/test_replication_from_dockerhub.py b/tests/apitests/python/test_replication_from_dockerhub.py index 29fd6c2c5..6fd3b4b73 100644 --- a/tests/apitests/python/test_replication_from_dockerhub.py +++ b/tests/apitests/python/test_replication_from_dockerhub.py @@ -94,7 +94,7 @@ class TestProjects(unittest.TestCase): self.replication.trigger_replication_executions(TestProjects.rule_id, **ADMIN_CLIENT) #7. Wait for completion of this replication job; - self.replication.wait_until_jobs_finish(TestProjects.rule_id, **ADMIN_CLIENT) + self.replication.wait_until_jobs_finish(TestProjects.rule_id,interval=30, **ADMIN_CLIENT) #8. Check image is replicated into target project successfully. artifact = self.artifact.get_reference_info(TestProjects.project_name, self.image, self.tag, **ADMIN_CLIENT) diff --git a/tests/apitests/python/update_docker_cfg.sh b/tests/apitests/python/update_docker_cfg.sh index 0ab4968ed..2e26e9007 100755 --- a/tests/apitests/python/update_docker_cfg.sh +++ b/tests/apitests/python/update_docker_cfg.sh @@ -1,5 +1,7 @@ #!/bin/sh -sudo sed -i '$d' /$HOME/.docker/config.json -sudo sed -i '$d' /$HOME/.docker/config.json -sudo echo -e "\n },\n \"experimental\": \"enabled\"\n}" >> /$HOME/.docker/config.json +if [ $(cat /$HOME/.docker/config.json |grep experimental |wc -l) -eq 0 ];then + sudo sed -i '$d' /$HOME/.docker/config.json + sudo sed -i '$d' /$HOME/.docker/config.json + sudo echo -e "},\n \"experimental\": \"enabled\"\n}" >> /$HOME/.docker/config.json +fi diff --git a/tests/ci/api_common_install.sh b/tests/ci/api_common_install.sh index a54c8c156..a894949ca 100755 --- a/tests/ci/api_common_install.sh +++ b/tests/ci/api_common_install.sh @@ -10,26 +10,13 @@ set -e if [ -z "$1" ]; then echo no ip specified; exit 1;fi # prepare cert ... sudo ./tests/generateCerts.sh $1 -sudo mkdir -p /etc/docker/certs.d/$1 && sudo cp ./tests/harbor_ca.crt /etc/docker/certs.d/$1/ && rm -rf ~/.docker/ && mkdir -p ~/.docker/tls/$1:4443/ && sudo cp ./tests/harbor_ca.crt ~/.docker/tls/$1:4443/ - +sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests --upgrade sudo ./tests/hostcfg.sh if [ "$2" = 'LDAP' ]; then cd tests && sudo ./ldapprepare.sh && cd .. fi -# prepare a chart file for API_DB test... -sudo curl -o $DIR/../../tests/apitests/python/mariadb-4.3.1.tgz https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz - -sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.2.1 robotframework-httplibrary requests --upgrade -sudo make swagger_client -#TODO: Swagger python package used to installed into dist-packages, but it's changed into site-packages all in a sudden, we havn't found the root cause. -# so current workround is to copy swagger packages from site-packages to dist-packages. -package_dir=/usr/lib/python3.7/site-packages -if [ -d $package_dir ] && [ $(find $package_dir -type f -name "*client*.egg" | wc -l) -gt 0 ];then - sudo cp -rf ${package_dir}/* /usr/local/lib/python3.7/dist-packages -fi - if [ $GITHUB_TOKEN ]; then sed "s/# github_token: xxx/github_token: $GITHUB_TOKEN/" -i make/harbor.yml diff --git a/tests/ci/api_run.sh b/tests/ci/api_run.sh index 33ad774bf..b4e14ddaa 100755 --- a/tests/ci/api_run.sh +++ b/tests/ci/api_run.sh @@ -30,7 +30,7 @@ set +e docker ps # run db auth api cases if [ "$1" = 'DB' ]; then - robot -X -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_DB.robot + docker run -i --privileged -v $DIR/../../:/drone -v $DIR/../:/ca -w /drone goharbor/harbor-e2e-engine:2.5 robot -v ip:$2 -v ip1: -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot /drone/tests/robot-cases/Group0-BAT/API_DB.robot elif [ "$1" = 'LDAP' ]; then # run ldap api cases python $DIR/../../tests/configharbor.py -H $IP -u $HARBOR_ADMIN -p $HARBOR_ADMIN_PASSWD -c auth_mode=ldap_auth \ @@ -39,7 +39,7 @@ elif [ "$1" = 'LDAP' ]; then ldap_search_password=admin \ ldap_base_dn=dc=example,dc=com \ ldap_uid=cn - robot -X -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_LDAP.robot + docker run -i --privileged -v $DIR/../../:/drone -v $DIR/../:/ca -w /drone goharbor/harbor-e2e-engine:2.5 robot -v ip:$2 -v ip1: -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot /drone/tests/robot-cases/Group0-BAT/API_LDAP.robot else rc=999 fi diff --git a/tests/e2e-image/nightly/Dockerfile b/tests/e2e-image/Dockerfile similarity index 65% rename from tests/e2e-image/nightly/Dockerfile rename to tests/e2e-image/Dockerfile index 1602bb0f2..10730d81d 100644 --- a/tests/e2e-image/nightly/Dockerfile +++ b/tests/e2e-image/Dockerfile @@ -1,24 +1,26 @@ -FROM golang:1.11.13 +FROM ubuntu:18.04 +ENV LANG C.UTF-8 +# V 2.0 +# V 2.0.1: upgrade docker to version 19.03.12 +# V 2.5 Add support for e2e py-test -#docker build -f drone/Dockerfile -t goharbor/harbor-e2e-engine:2.4 . -# V2.4.1: Upgrade docker engine -RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - +RUN apt-get update && apt-get install -y --no-install-recommends wget curl gnupg2 +RUN wget --no-check-certificate -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - RUN sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' -#RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && apt install ./google-chrome-stable_current_amd64.deb - RUN apt-get update && apt-get install -y --no-install-recommends \ jq \ bc \ time \ gcc \ + git \ python-dev \ libffi-dev \ libssl-dev \ sshpass \ ant \ - ant-optional \ - xvfb \ + ant-optional \ + xvfb \ libxi6 \ libgconf-2-4 \ openjdk-8-jdk \ @@ -27,7 +29,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ unzip \ zip \ bzip2 \ - parted \ + parted \ + #ip tool + #ethtool \ + iproute2 \ + #bridge-utils \ + #iputils-ping \ # Add docker in docker support btrfs-tools \ e2fsprogs \ @@ -46,19 +53,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ xz-utils \ xorriso \ sendmail && \ - # Cleanup + # Cleanup apt-get autoremove -y && \ rm -rf /var/lib/apt/lists/* -RUN sed -i s@/archive.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list && apt-get clean -RUN apt-get update -y && \ - apt-get install -y zbar-tools libzbar-dev python-zbar -RUN dpkg -L libzbar-dev; ls -l /usr/include/zbar.h -RUN apt-get update -y -RUN apt-get install -y python3.5 -RUN rm /usr/bin/python -RUN ln -s /usr/bin/python3.5 /usr/bin/python -RUN apt-get install -y python3-pip +RUN apt-get update && apt-get install -y software-properties-common && \ + add-apt-repository -y ppa:longsleep/golang-backports +RUN apt-get update && \ + apt-get install -y golang-go + +RUN apt-get update -y ; apt-get install -y zbar-tools libzbar-dev python-zbar python3.7 +RUN rm /usr/bin/python ; ln -s /usr/bin/python3.7 /usr/bin/python ; apt-get install -y python3-pip +RUN python -m pip install --upgrade pip RUN wget -N http://chromedriver.storage.googleapis.com/2.40/chromedriver_linux64.zip && \ unzip chromedriver_linux64.zip && \ @@ -72,9 +78,7 @@ RUN apt-get update && apt install libnss3-tools && \ echo Harbor12345 > password.ca && \ certutil -d sql:$HOME/.pki/nssdb -N -f password.ca -RUN wget https://bootstrap.pypa.io/get-pip.py && \ - python ./get-pip.py && \ - pip install pyasn1 google-apitools==0.5.15 gsutil robotframework==3.2.1 robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-seleniumlibrary==4.3.0 robotframework-pabot robotframework-JSONLibrary --upgrade +RUN pip3 install pyasn1 google-apitools==0.5.31 gsutil robotframework==3.2.1 robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-seleniumlibrary==4.3.0 robotframework-pabot robotframework-JSONLibrary --upgrade # Install docker, docker compose RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-19.03.12.tgz && \ @@ -83,19 +87,31 @@ RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-19.03.12. chmod +x /usr/local/bin/docker-compose RUN wget https://github.com/drone/drone-cli/releases/download/v0.8.3/drone_linux_amd64.tar.gz && tar zxf drone_linux_amd64.tar.gz && \ - install -t /usr/local/bin drone + install -t /usr/local/bin drone && mv drone drone_src + +RUN set -x \ + && groupadd --system dockremap \ + && adduser --system --ingroup dockremap dockremap \ + && echo 'dockremap:165536:65536' >> /etc/subuid \ + && echo 'dockremap:165536:65536' >> /etc/subgid RUN curl -sSL https://github.com/vmware/govmomi/releases/download/v0.16.0/govc_linux_amd64.gz | gzip -d > /usr/local/bin/govc && \ chmod +x /usr/local/bin/govc RUN wget https://get.helm.sh/helm-v2.14.1-linux-386.tar.gz && tar zxvf helm-v2.14.1-linux-386.tar.gz && \ - mv linux-386/helm /usr/local/bin/helm && \ - helm init --client-only && \ - helm plugin install https://github.com/chartmuseum/helm-push + cp linux-386/helm /usr/local/bin/helm && \ + cp linux-386/helm /usr/local/bin/helm2 && \ + helm init --client-only && \ + helm plugin install https://github.com/chartmuseum/helm-push RUN wget https://get.helm.sh/helm-v3.0.0-linux-386.tar.gz && tar zxvf helm-v3.0.0-linux-386.tar.gz && \ mv linux-386/helm /usr/local/bin/helm3 && \ - helm3 plugin install https://github.com/chartmuseum/helm-push + helm3 plugin install https://github.com/chartmuseum/helm-push + +RUN curl -LO https://github.com/deislabs/oras/releases/download/v0.8.1/oras_0.8.1_linux_amd64.tar.gz && \ + mkdir -p oras-install/ && \ + tar -zxf oras_0.8.1_*.tar.gz -C oras-install/ && \ + mv oras-install/oras /usr/local/bin/ RUN wget https://github.com/theupdateframework/notary/releases/download/v0.6.1/notary-Linux-amd64 && \ chmod +x notary-Linux-amd64 && \ @@ -127,23 +143,19 @@ RUN CNAB_PATH=$(go env GOPATH)/src/github.com/docker && mkdir -p $CNAB_PATH && c make build && \ mv bin/cnab-to-oci /usr/local/bin -# Add docker in docker support -# version: docker:1.13-dind -# reference: https://github.com/docker-library/docker/blob/b202ec7e529f5426e2ad7e8c0a8b82cacd406573/1.13/dind/Dockerfile -# -# https://github.com/docker/docker/blob/master/project/PACKAGERS.md#runtime-dependencies +RUN apt-get install -y sudo uuid-dev -# set up subuid/subgid so that "--userns-remap=default" works out-of-the-box -RUN set -x \ - && groupadd --system dockremap \ - && adduser --system --ingroup dockremap dockremap \ - && echo 'dockremap:165536:65536' >> /etc/subuid \ - && echo 'dockremap:165536:65536' >> /etc/subgid +RUN wget https://github.com/sylabs/singularity/releases/download/v3.3.0/singularity-3.3.0.tar.gz && \ + tar -xzf singularity-3.3.0.tar.gz && \ + cd singularity && \ + ./mconfig && \ + make -C builddir && \ + make -C builddir install ENV DIND_COMMIT 3b5fac462d21ca164b3778647420016315289034 RUN wget "https://raw.githubusercontent.com/docker/docker/${DIND_COMMIT}/hack/dind" -O /usr/local/bin/dind \ - && chmod +x /usr/local/bin/dind + && chmod +x /usr/local/bin/dind # This container needs to be run in privileged mode(run with --privileged option) to make it work COPY dockerd-entrypoint.sh /usr/local/bin/dockerd-entrypoint.sh diff --git a/tests/e2e-image/dockerd-entrypoint.sh b/tests/e2e-image/dockerd-entrypoint.sh index 98bb62ee3..348541b0f 100644 --- a/tests/e2e-image/dockerd-entrypoint.sh +++ b/tests/e2e-image/dockerd-entrypoint.sh @@ -38,6 +38,7 @@ if [ "$1" = 'dockerd' ]; then set -- sh "$(which dind)" "$@" "--insecure-registry=0.0.0.0/0" fi +containerd & echo "$@" exec "$@" diff --git a/tests/resources/Helm-Util.robot b/tests/resources/Helm-Util.robot index 6ca70de42..31f91263d 100644 --- a/tests/resources/Helm-Util.robot +++ b/tests/resources/Helm-Util.robot @@ -18,9 +18,6 @@ Library OperatingSystem Library Process *** Keywords *** -Prepare Helm Cert - Wait Unitl Command Success cp harbor_ca.crt /ca/server.crt - Helm Repo Add [Arguments] ${harbor_url} ${user} ${pwd} ${project_name}=library ${helm_repo_name}=myrepo ${rc} ${output}= Run And Return Rc And Output helm repo remove ${project_name} diff --git a/tests/resources/Nightly-Util.robot b/tests/resources/Nightly-Util.robot index dcc80023e..2084aff33 100644 --- a/tests/resources/Nightly-Util.robot +++ b/tests/resources/Nightly-Util.robot @@ -24,15 +24,21 @@ Nightly Test Setup [Arguments] ${ip} ${HARBOR_PASSWORD} ${ip1}==${EMPTY} Run Keyword If '${ip1}' != '${EMPTY}' CA setup ${ip1} ${HARBOR_PASSWORD} /ca/ca1.crt Run Keyword If '${ip1}' != '${EMPTY}' Run rm -rf ./harbor_ca.crt + Log To Console CA setup ... Run Keyword CA setup ${ip} ${HARBOR_PASSWORD} + Log To Console Start Docker Daemon Locally ... Run Keyword Start Docker Daemon Locally + Log To Console wget mariadb ... + Run wget ${prometheus_chart_file_url} CA Setup [Arguments] ${ip} ${HARBOR_PASSWORD} ${cert}=/ca/ca.crt - Run cp ${cert} harbor_ca.crt + Log To Console cp /ca/harbor_ca.crt harbor_ca.crt ... + Run cp /ca/harbor_ca.crt harbor_ca.crt + Log To Console Generate Certificate Authority For Chrome ... Generate Certificate Authority For Chrome ${HARBOR_PASSWORD} + Log To Console Prepare Docker Cert ... Prepare Docker Cert ${ip} - Prepare Helm Cert Collect Nightly Logs [Arguments] ${ip} ${SSH_PWD} ${ip1}==${EMPTY}