Create Dockerfile for E2E py-test

Robot tests have been always executed in docker container, but API pyt-test is not,
so rebuild Dockerfile for it. During that, some of tools have been upgraded, so API py-test
 scripts need some modification.

Signed-off-by: danfengliu <danfengl@vmware.com>

.github/workflows/CI.yml

@@ -113,7 +113,6 @@ jobs:
           cd src/github.com/goharbor/harbor
           pwd
           go env
-          echo "::set-env name=CNAB_PATH::$(go env GOPATH)/src/github.com/docker"
           echo "::set-env name=GITHUB_TOKEN::${{ secrets.GITHUB_TOKEN }}"
           echo "::set-env name=GOPATH::$(go env GOPATH):$GITHUB_WORKSPACE"
           echo "::add-path::$(go env GOPATH)/bin"
@@ -133,40 +132,6 @@ jobs:
           curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
           chmod +x docker-compose
           sudo mv docker-compose /usr/local/bin
-          echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
-          sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
-          sudo update-ca-certificates
-          sudo service docker restart
-          wget https://get.helm.sh/helm-v2.14.1-linux-386.tar.gz && tar zxvf helm-v2.14.1-linux-386.tar.gz
-          sudo mv linux-386/helm /usr/local/bin/helm2
-          helm2 init --client-only
-          helm2 plugin list | grep push || helm2 plugin install https://github.com/chartmuseum/helm-push
-          wget https://get.helm.sh/helm-v3.1.1-linux-386.tar.gz && tar zxvf helm-v3.1.1-linux-386.tar.gz
-          sudo mv linux-386/helm /usr/local/bin/helm3
-          helm3 plugin list | grep push || helm3 plugin install https://github.com/chartmuseum/helm-push
-          rm -rf $CNAB_PATH;mkdir -p $CNAB_PATH && cd $CNAB_PATH && git clone https://github.com/cnabio/cnab-to-oci.git
-          cd cnab-to-oci && git checkout v0.3.0-beta4
-          go list
-          make build
-          sudo mv bin/cnab-to-oci /usr/local/bin
-          curl -LO https://github.com/deislabs/oras/releases/download/v0.8.1/oras_0.8.1_linux_amd64.tar.gz
-          mkdir -p oras-install/
-          tar -zxf oras_0.8.1_*.tar.gz -C oras-install/
-          sudo mv oras-install/oras /usr/local/bin/
-          sudo apt-get update && sudo apt-get install -y \
-          build-essential \
-          uuid-dev \
-          libgpgme-dev \
-          squashfs-tools \
-          libseccomp-dev \
-          pkg-config \
-          cryptsetup-bin
-          export VERSION=3.5.3 && \
-          wget https://github.com/sylabs/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz && \
-          tar -xzf singularity-${VERSION}.tar.gz && cd singularity
-          ./mconfig && make -C builddir && sudo make -C builddir install
-          sudo apt-get update -y ; sudo apt-get install -y zbar-tools libzbar-dev python-zbar python3.7
-          sudo rm /usr/bin/python ; sudo ln -s /usr/bin/python3.7 /usr/bin/python ; sudo apt-get install -y python3-pip
       - name: install
         run: |
           cd src/github.com/goharbor/harbor
@@ -211,6 +176,8 @@ jobs:
           echo "::set-env name=GOPATH::$(go env GOPATH):$GITHUB_WORKSPACE"
           echo "::add-path::$(go env GOPATH)/bin"
           echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/src/github.com/goharbor/harbor/tests/private_key.pem"
+          IP=`hostname -I | awk '{print $1}'`
+          echo "::set-env name=IP::$IP"
         shell: bash
       - name: before_install
         run: |
@@ -224,12 +191,6 @@ jobs:
           curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
           chmod +x docker-compose
           sudo mv docker-compose /usr/local/bin
-          IP=`hostname -I | awk '{print $1}'`
-          echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
-          echo "::set-env name=IP::$IP"
-          python -V
-          sudo apt-get update -y ; sudo apt-get install -y zbar-tools libzbar-dev python-zbar python3.7
-          sudo rm /usr/bin/python ; sudo ln -s /usr/bin/python3.7 /usr/bin/python ; sudo apt-get install -y python3-pip
       - name: install
         run: |
           cd src/github.com/goharbor/harbor

tests/apitests/python/library/singularity.py

@@ -15,7 +15,7 @@ def singularity_push_to_harbor(harbor_server, sif_file, project, image, tag):
     ret = base.run_command( [singularity_cmd, "push", sif_file, "oras://"+harbor_server + "/" + project + "/" + image+":"+ tag] )
 
 def singularity_pull(out_file, from_sif_file):
-    ret = base.run_command( [singularity_cmd, "pull", out_file, from_sif_file] )
+    ret = base.run_command( [singularity_cmd, "pull", "--allow-unsigned", out_file, from_sif_file] )
 
 def push_singularity_to_harbor(from_URI, from_namespace, harbor_server, user, password, project, image, tag):
     tmp_sif_file = image+timestamp+".sif"

tests/apitests/python/test_list_helm_charts.py

@@ -4,6 +4,7 @@ import unittest
 
 from testutils import ADMIN_CLIENT, CHART_API_CLIENT
 from testutils import TEARDOWN
+import base
 from library.user import User
 from library.project import Project
 from library.chart import Chart
@@ -50,6 +51,7 @@ class TestProjects(unittest.TestCase):
         TestProjects.CHART_NAME = 'mariadb'
         TestProjects.VERSION = '4.3.1'
 
+        base.run_command( ["curl", r"-o", "./tests/apitests/python/mariadb-4.3.1.tgz", "https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz"])
         #1. Create a new user(UA);
         TestProjects.user_chart_id, user_chart_name = self.user.create_user(user_password = user_chart_password, **ADMIN_CLIENT)
 

tests/apitests/python/test_replication_from_dockerhub.py

@@ -94,7 +94,7 @@ class TestProjects(unittest.TestCase):
         self.replication.trigger_replication_executions(TestProjects.rule_id, **ADMIN_CLIENT)
 
         #7. Wait for completion of this replication job;
-        self.replication.wait_until_jobs_finish(TestProjects.rule_id, **ADMIN_CLIENT)
+        self.replication.wait_until_jobs_finish(TestProjects.rule_id,interval=30, **ADMIN_CLIENT)
 
         #8. Check image is replicated into target project successfully.
         artifact = self.artifact.get_reference_info(TestProjects.project_name, self.image, self.tag, **ADMIN_CLIENT)

tests/apitests/python/update_docker_cfg.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
-sudo sed -i '$d' /$HOME/.docker/config.json
+if [ $(cat /$HOME/.docker/config.json |grep experimental |wc -l) -eq 0 ];then
-sudo sed -i '$d' /$HOME/.docker/config.json
+    sudo sed -i '$d' /$HOME/.docker/config.json
-sudo echo -e "\n        },\n        \"experimental\": \"enabled\"\n}" >> /$HOME/.docker/config.json
+    sudo sed -i '$d' /$HOME/.docker/config.json
+    sudo echo -e "},\n        \"experimental\": \"enabled\"\n}" >> /$HOME/.docker/config.json
+fi

tests/ci/api_common_install.sh

@@ -10,26 +10,13 @@ set -e
 if [ -z "$1" ]; then echo no ip specified; exit 1;fi
 # prepare cert ...
 sudo ./tests/generateCerts.sh $1
-sudo mkdir -p /etc/docker/certs.d/$1 && sudo cp ./tests/harbor_ca.crt /etc/docker/certs.d/$1/ && rm -rf ~/.docker/ &&  mkdir -p ~/.docker/tls/$1:4443/ && sudo cp ./tests/harbor_ca.crt ~/.docker/tls/$1:4443/
+sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests --upgrade
-
 sudo ./tests/hostcfg.sh
 
 if [ "$2" = 'LDAP' ]; then
     cd tests && sudo ./ldapprepare.sh && cd ..
 fi
 
-# prepare a chart file for API_DB test...
-sudo curl -o $DIR/../../tests/apitests/python/mariadb-4.3.1.tgz https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz
-
-sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.2.1 robotframework-httplibrary requests --upgrade
-sudo make swagger_client
-#TODO: Swagger python package used to installed into dist-packages, but it's changed into site-packages all in a sudden, we havn't found the root cause.
-#      so current workround is to copy swagger packages from site-packages to dist-packages.
-package_dir=/usr/lib/python3.7/site-packages
-if [ -d $package_dir ] && [  $(find $package_dir -type f -name "*client*.egg" | wc -l) -gt 0 ];then
-    sudo cp -rf ${package_dir}/* /usr/local/lib/python3.7/dist-packages
-fi
-
 if [ $GITHUB_TOKEN ];
 then
     sed "s/# github_token: xxx/github_token: $GITHUB_TOKEN/" -i make/harbor.yml

tests/ci/api_run.sh

@@ -30,7 +30,7 @@ set +e
 docker ps
 # run db auth api cases
 if [ "$1" = 'DB' ]; then
-    robot -X -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_DB.robot
+    docker run -i --privileged -v $DIR/../../:/drone -v $DIR/../:/ca -w /drone goharbor/harbor-e2e-engine:2.5 robot -v ip:$2  -v ip1: -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot /drone/tests/robot-cases/Group0-BAT/API_DB.robot
 elif [ "$1" = 'LDAP' ]; then
     # run ldap api cases
     python $DIR/../../tests/configharbor.py -H $IP -u $HARBOR_ADMIN -p $HARBOR_ADMIN_PASSWD -c auth_mode=ldap_auth \
@@ -39,7 +39,7 @@ elif [ "$1" = 'LDAP' ]; then
                                   ldap_search_password=admin \
                                   ldap_base_dn=dc=example,dc=com \
                                   ldap_uid=cn
-    robot -X -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_LDAP.robot
+    docker run -i --privileged -v $DIR/../../:/drone -v $DIR/../:/ca -w /drone goharbor/harbor-e2e-engine:2.5 robot -v ip:$2  -v ip1: -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot /drone/tests/robot-cases/Group0-BAT/API_LDAP.robot
 else
     rc=999
 fi

tests/e2e-image/Dockerfile

@@ -1,24 +1,26 @@
-FROM golang:1.11.13
+FROM ubuntu:18.04
+ENV LANG C.UTF-8
+# V 2.0
+# V 2.0.1: upgrade docker to version 19.03.12
+# V 2.5 Add support for e2e py-test 
 
-#docker build -f drone/Dockerfile -t goharbor/harbor-e2e-engine:2.4 .
+RUN apt-get update && apt-get install -y --no-install-recommends wget curl gnupg2
-# V2.4.1: Upgrade docker engine
+RUN wget  --no-check-certificate -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
-RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
 RUN sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
 
-#RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && apt install ./google-chrome-stable_current_amd64.deb
-
 RUN apt-get update && apt-get install -y --no-install-recommends \
     jq \
     bc \
     time \
     gcc \
+    git \
     python-dev \
     libffi-dev \
     libssl-dev \
     sshpass \
     ant \
-	ant-optional \
+    ant-optional \
-	xvfb \
+    xvfb \
     libxi6 \
     libgconf-2-4 \
     openjdk-8-jdk \
@@ -27,7 +29,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     unzip \
     zip \
     bzip2 \
-	parted \
+    parted \
+    #ip tool
+    #ethtool \
+    iproute2 \
+    #bridge-utils \
+    #iputils-ping \
     # Add docker in docker support
     btrfs-tools \
     e2fsprogs \
@@ -46,19 +53,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     xz-utils \
     xorriso \
     sendmail && \
-	# Cleanup
+    # Cleanup
     apt-get autoremove -y && \
     rm -rf /var/lib/apt/lists/*
 
-RUN sed -i s@/archive.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list && apt-get clean
+RUN apt-get update && apt-get install -y software-properties-common && \
-RUN apt-get update -y && \
+    add-apt-repository -y ppa:longsleep/golang-backports
-	apt-get install -y zbar-tools libzbar-dev python-zbar
+RUN apt-get update && \
-RUN dpkg -L libzbar-dev; ls -l /usr/include/zbar.h
+    apt-get install -y  golang-go
-RUN apt-get update -y
+
-RUN apt-get install -y python3.5
+RUN apt-get update -y ; apt-get install -y zbar-tools libzbar-dev python-zbar python3.7
-RUN rm /usr/bin/python
+RUN rm /usr/bin/python ; ln -s /usr/bin/python3.7 /usr/bin/python ; apt-get install -y python3-pip
-RUN ln -s /usr/bin/python3.5 /usr/bin/python
+RUN python -m pip install --upgrade pip
-RUN apt-get install -y python3-pip
 
 RUN wget -N http://chromedriver.storage.googleapis.com/2.40/chromedriver_linux64.zip && \
     unzip chromedriver_linux64.zip && \
@@ -72,9 +78,7 @@ RUN apt-get update && apt install libnss3-tools && \
     echo Harbor12345 > password.ca && \
     certutil -d sql:$HOME/.pki/nssdb -N -f password.ca
 
-RUN wget https://bootstrap.pypa.io/get-pip.py && \
+RUN pip3 install pyasn1 google-apitools==0.5.31 gsutil robotframework==3.2.1 robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-seleniumlibrary==4.3.0 robotframework-pabot robotframework-JSONLibrary --upgrade
-    python ./get-pip.py  && \
-    pip install pyasn1 google-apitools==0.5.15 gsutil robotframework==3.2.1 robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-seleniumlibrary==4.3.0 robotframework-pabot robotframework-JSONLibrary --upgrade
 
 # Install docker, docker compose
 RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-19.03.12.tgz && \
@@ -83,19 +87,31 @@ RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-19.03.12.
     chmod +x /usr/local/bin/docker-compose
 
 RUN wget https://github.com/drone/drone-cli/releases/download/v0.8.3/drone_linux_amd64.tar.gz && tar zxf drone_linux_amd64.tar.gz && \
-    install -t /usr/local/bin drone
+    install -t /usr/local/bin drone && mv drone drone_src
+
+RUN set -x \
+        && groupadd --system dockremap \
+        && adduser --system --ingroup dockremap dockremap \
+        && echo 'dockremap:165536:65536' >> /etc/subuid \
+        && echo 'dockremap:165536:65536' >> /etc/subgid
 
 RUN curl -sSL https://github.com/vmware/govmomi/releases/download/v0.16.0/govc_linux_amd64.gz | gzip -d > /usr/local/bin/govc && \
     chmod +x /usr/local/bin/govc
 
 RUN wget https://get.helm.sh/helm-v2.14.1-linux-386.tar.gz && tar zxvf helm-v2.14.1-linux-386.tar.gz && \
-    mv linux-386/helm /usr/local/bin/helm && \
+    cp linux-386/helm /usr/local/bin/helm && \
-	helm init --client-only && \
+    cp linux-386/helm /usr/local/bin/helm2 && \
-	helm plugin install https://github.com/chartmuseum/helm-push
+    helm init --client-only && \
+    helm plugin install https://github.com/chartmuseum/helm-push
 
 RUN wget https://get.helm.sh/helm-v3.0.0-linux-386.tar.gz && tar zxvf helm-v3.0.0-linux-386.tar.gz && \
     mv linux-386/helm /usr/local/bin/helm3 && \
-	helm3 plugin install https://github.com/chartmuseum/helm-push
+    helm3 plugin install https://github.com/chartmuseum/helm-push
+
+RUN curl -LO https://github.com/deislabs/oras/releases/download/v0.8.1/oras_0.8.1_linux_amd64.tar.gz && \
+    mkdir -p oras-install/  && \
+    tar -zxf oras_0.8.1_*.tar.gz -C oras-install/  && \
+    mv oras-install/oras /usr/local/bin/
 
 RUN wget https://github.com/theupdateframework/notary/releases/download/v0.6.1/notary-Linux-amd64 && \
     chmod +x notary-Linux-amd64 && \
@@ -127,23 +143,19 @@ RUN CNAB_PATH=$(go env GOPATH)/src/github.com/docker && mkdir -p $CNAB_PATH && c
     make build && \
     mv bin/cnab-to-oci /usr/local/bin
 
-# Add docker in docker support
+RUN apt-get install -y sudo uuid-dev
-# version: docker:1.13-dind
-# reference: https://github.com/docker-library/docker/blob/b202ec7e529f5426e2ad7e8c0a8b82cacd406573/1.13/dind/Dockerfile
-#
-# https://github.com/docker/docker/blob/master/project/PACKAGERS.md#runtime-dependencies
 
-# set up subuid/subgid so that "--userns-remap=default" works out-of-the-box
+RUN wget https://github.com/sylabs/singularity/releases/download/v3.3.0/singularity-3.3.0.tar.gz && \
-RUN set -x \
+    tar -xzf singularity-3.3.0.tar.gz && \
-        && groupadd --system dockremap \
+    cd singularity && \
-        && adduser --system --ingroup dockremap dockremap \
+    ./mconfig && \
-        && echo 'dockremap:165536:65536' >> /etc/subuid \
+    make -C builddir && \
-        && echo 'dockremap:165536:65536' >> /etc/subgid
+    make -C builddir install
 
 ENV DIND_COMMIT 3b5fac462d21ca164b3778647420016315289034
 
 RUN wget "https://raw.githubusercontent.com/docker/docker/${DIND_COMMIT}/hack/dind" -O /usr/local/bin/dind \
-        && chmod +x /usr/local/bin/dind
+    && chmod +x /usr/local/bin/dind
 
 # This container needs to be run in privileged mode(run with --privileged option) to make it work
 COPY dockerd-entrypoint.sh /usr/local/bin/dockerd-entrypoint.sh

tests/e2e-image/dockerd-entrypoint.sh

@@ -38,6 +38,7 @@ if [ "$1" = 'dockerd' ]; then
 	set -- sh "$(which dind)" "$@" "--insecure-registry=0.0.0.0/0"
 fi
 
+containerd &
 
 echo "$@"
 exec "$@"

tests/resources/Helm-Util.robot

@@ -18,9 +18,6 @@ Library  OperatingSystem
 Library  Process
 
 *** Keywords ***
-Prepare Helm Cert
-    Wait Unitl Command Success  cp harbor_ca.crt /ca/server.crt
-
 Helm Repo Add
     [Arguments]  ${harbor_url}  ${user}  ${pwd}  ${project_name}=library  ${helm_repo_name}=myrepo
     ${rc}  ${output}=  Run And Return Rc And Output  helm repo remove ${project_name}

tests/resources/Nightly-Util.robot

@@ -24,15 +24,21 @@ Nightly Test Setup
     [Arguments]  ${ip}  ${HARBOR_PASSWORD}  ${ip1}==${EMPTY}
     Run Keyword If  '${ip1}' != '${EMPTY}'  CA setup  ${ip1}  ${HARBOR_PASSWORD}  /ca/ca1.crt
     Run Keyword If  '${ip1}' != '${EMPTY}'  Run  rm -rf ./harbor_ca.crt
+    Log To Console  CA setup ...
     Run Keyword  CA setup  ${ip}  ${HARBOR_PASSWORD}
+    Log To Console  Start Docker Daemon Locally ...
     Run Keyword  Start Docker Daemon Locally
+    Log To Console  wget mariadb ...
+    Run  wget ${prometheus_chart_file_url}
 
 CA Setup
     [Arguments]  ${ip}  ${HARBOR_PASSWORD}  ${cert}=/ca/ca.crt
-    Run  cp ${cert} harbor_ca.crt
+    Log To Console  cp /ca/harbor_ca.crt harbor_ca.crt ...
+    Run  cp /ca/harbor_ca.crt harbor_ca.crt
+    Log To Console  Generate Certificate Authority For Chrome ...
     Generate Certificate Authority For Chrome  ${HARBOR_PASSWORD}
+    Log To Console  Prepare Docker Cert ...
     Prepare Docker Cert  ${ip}
-    Prepare Helm Cert
 
 Collect Nightly Logs
     [Arguments]  ${ip}  ${SSH_PWD}  ${ip1}==${EMPTY}