Remove deploytment of python3.6 and set dns for docker

1. Docker v20 is the default version for git action, dns should be set manually.
Python v3.8 is the default version for git action, so remove deployment for python v3.6.
2. Some of API tests would be affect by docker v20, like manifest get
message will not be triggered if there is one locally, to avoid this
impact, API tests will be tested with docker E2E image, in this image,
docker v19 is enabled.

Signed-off-by: danfengliu <danfengl@vmware.com>

tests/apitests/python/library/base.py

@@ -2,6 +2,7 @@
 
 import sys
 import time
+import subprocess
 import swagger_client
 try:
     from urllib import getproxies
@@ -60,6 +61,24 @@ def _get_string_from_unicode(udata):
         result = result + tmp.strip('\n\r\t')
     return result
 
+def run_command(command, expected_error_message = None):
+    print("Command: ", subprocess.list2cmdline(command))
+    try:
+        output = subprocess.check_output(command,
+                                         stderr=subprocess.STDOUT,
+                                         universal_newlines=True)
+    except subprocess.CalledProcessError as e:
+        print("Run command error:", str(e))
+        print("expected_error_message:", expected_error_message)
+        if expected_error_message is not None:
+            if str(e.output).lower().find(expected_error_message.lower()) < 0:
+                raise Exception(r"Error message {} is not as expected {}".format(str(e.output), expected_error_message))
+        else:
+            raise Exception('Error: Exited with error code: %s. Output:%s'% (e.returncode, e.output))
+    else:
+        print("output:", output)
+        return
+
 class Base:
     def __init__(self,
         server = Server(endpoint="http://localhost:8080/api", verify_ssl=False),

tests/apitests/python/library/docker_api.py

@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
 
 import base
+import subprocess
+import json
+from testutils import DOCKER_USER, DOCKER_PWD, BASE_IMAGE, BASE_IMAGE_ABS_PATH_NAME
 
 try:
     import docker
@@ -8,9 +11,86 @@ except ImportError:
     import pip
     pip.main(['install', 'docker'])
     import docker
+
+def docker_info_display():
+    command = ["docker", "info", "-f", "'{{.OSType}}/{{.Architecture}}'"]
+    print("Docker Info: ", command)
+    ret = base.run_command(command)
+    print("Command return: ", ret)
+
+def docker_login_cmd(harbor_host, username, password, cfg_file = "./tests/apitests/python/update_docker_cfg.sh",  enable_manifest = True):
+    if  username == "" or password == "":
+        print("[Warnig]: No docker credential was provided.")
+        return
+    command = ["sudo", "docker", "login", harbor_host, "-u", username, "-p", password]
+    print( "Docker Login Command: ", command)
+    base.run_command(command)
+    if enable_manifest == True:
+        try:
+            ret = subprocess.check_output([cfg_file], shell=False)
+            print("docker login cmd ret:", ret)
+        except subprocess.CalledProcessError as exc:
+            raise Exception("Failed to update docker config, error is {} {}.".format(exc.returncode, exc.output))
+
+def docker_manifest_create(index, manifests):
+    command = ["sudo", "docker","manifest","create", "--amend", index]
+    command.extend(manifests)
+    print( "Docker Manifest Command: ", command)
+    base.run_command(command)
+
+def docker_images_all_list():
+    command = ["sudo", "docker","images","-a"]
+    base.run_command(command)
+
+def docker_load_image(image):
+    command = ["sudo", "docker","load","-i", image]
+    base.run_command(command)
+
+def docker_manifest_push(index):
+    command = ["sudo", "docker","manifest","push",index]
+    print( "Docker Manifest Command: ", command)
+    ret = base.run_command(command)
+    index_sha256=""
+    manifest_list=[]
+    for line in ret.split("\n"):
+        if line[:7] == "sha256:":
+            index_sha256 = line
+        if line.find('Pushed ref') == 0:
+            manifest_list.append(line[-71:])
+    return index_sha256, manifest_list
+
+def docker_manifest_push_to_harbor(index, manifests, harbor_server, username, password, cfg_file = "./tests/apitests/python/update_docker_cfg.sh"):
+    docker_login_cmd(harbor_server, username, password, cfg_file=cfg_file)
+    docker_manifest_create(index, manifests)
+    return docker_manifest_push(index)
+
+def list_repositories(harbor_host, username, password, n = None, last = None):
+    if n is not None and last is not None:
+        command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/_catalog"+"?n=%d"%n+"&last="+last, "--insecure"]
+    elif n is not None:
+            command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/_catalog"+"?n=%d"%n, "--insecure"]
+    else:
+        command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/_catalog", "--insecure"]
+    print( "List Repositories Command: ", command)
+    ret = base.run_command(command)
+    repos = json.loads(ret).get("repositories","")
+    return repos
+
+def list_image_tags(harbor_host, repository, username, password, n = None, last = None):
+    if n is not None and last is not None:
+        command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/"+repository+"/tags/list"+"?n=%d"%n+"&last="+last, "--insecure"]
+    elif n is not None:
+        command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/"+repository+"/tags/list"+"?n=%d"%n, "--insecure"]
+    else:
+        command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/"+repository+"/tags/list", "--insecure"]
+    print( "List Image Tags Command: ", command)
+    ret = base.run_command(command)
+    tags = json.loads(ret).get("tags","")
+    return tags
+
 class DockerAPI(object):
     def __init__(self):
-        self.DCLIENT = docker.APIClient(base_url='unix://var/run/docker.sock',version='auto',timeout=10)
+        self.DCLIENT = docker.APIClient(base_url='unix://var/run/docker.sock',version='auto',timeout=30)
         self.DCLIENT2 = docker.from_env()
 
     def docker_login(self, registry, username, password, expected_error_message = None):
@@ -37,12 +117,12 @@ class DockerAPI(object):
         finally:
             if expected_error_message is not None:
                 if str(err_message).lower().find(expected_error_message.lower()) < 0:
-                    raise Exception(r" Failed to catch error [{}] when login image {}, return message: {}".format (expected_error_message, image, err_message))
+                    raise Exception(r" Failed to catch error [{}] when login registry {}, return message: {}".format (expected_error_message, registry, err_message))
                 else:
                     print(r"Docker image login got expected error message:{}".format(expected_error_message))
             else:
                 if str(err_message).lower().find("error".lower()) >= 0:
-                    raise Exception(r" It's was not suppose to catch error when login image {}, return message is [{}]".format (image, err_message))
+                    raise Exception(r" It's was not suppose to catch error when login registry {}, return message is [{}]".format (registry, err_message))
 
     def docker_image_pull(self, image, tag = None, expected_error_message = None):
         ret = ""
@@ -88,6 +168,7 @@ class DockerAPI(object):
     def docker_image_push(self, harbor_registry, tag, expected_error_message = None):
         ret = ""
         err_message = ""
+        docker_images_all_list()
         if expected_error_message is "":
             expected_error_message = None
         try:
@@ -109,16 +190,17 @@ class DockerAPI(object):
             else:
                 if str(err_message).lower().find("error".lower()) >= 0:
                     raise Exception(r" It's was not suppose to catch error when push image {}, return message is [{}]".format (harbor_registry, err_message))
+        docker_images_all_list()
 
     def docker_image_build(self, harbor_registry, tags=None, size=1, expected_error_message = None):
         ret = ""
         err_message = ""
         try:
-            baseimage='busybox:latest'
+            baseimage = BASE_IMAGE['name'] + ":" + BASE_IMAGE['tag']
             if not self.DCLIENT.images(name=baseimage):
-                print( "Docker pull is triggered when building {}".format(harbor_registry))
+                print( "Docker load is triggered when building {}".format(harbor_registry))
-                self.DCLIENT.pull(baseimage)
+                docker_load_image(BASE_IMAGE_ABS_PATH_NAME)
-            c=self.DCLIENT.create_container(image='busybox:latest',
+            c = self.DCLIENT.create_container(image=baseimage,
                 command='dd if=/dev/urandom of=test bs=1M count={}'.format(size))
             self.DCLIENT.start(c)
             self.DCLIENT.wait(c)
@@ -135,10 +217,7 @@ class DockerAPI(object):
                 ret = self.DCLIENT.push(repo)
                 print("docker_image_push ret:", ret)
                 print("build image {} with size {}".format(repo, size))
-                self.DCLIENT.remove_image(repo)
             self.DCLIENT.remove_container(c)
-            #self.DCLIENT.pull(repo)
-            #image = self.DCLIENT2.images.get(repo)
         except Exception as err:
             print( "Docker image build catch exception:", str(err))
             err_message = str(err)
@@ -152,7 +231,7 @@ class DockerAPI(object):
                 if str(err_message).lower().find(expected_error_message.lower()) < 0:
                     raise Exception(r" Failed to catch error [{}] when build image {}, return message: {}".format (expected_error_message, harbor_registry, err_message))
                 else:
-                    print(r"Docker image build got expected error message:{}".format(expected_error_message))
+                    print(r"Docker image build got expected error message: {}".format(expected_error_message))
             else:
                 if str(err_message).lower().find("error".lower()) >= 0:
-                    raise Exception(r" It's was not suppose to catch error when build image {}, return message is [{}]".format (harbor_registry, err_message))
+                    raise Exception(r" It's was not suppose to catch error when build image {}, return message is [{}]".format (harbor_registry, err_message))

tests/apitests/python/test_list_helm_charts.py

@@ -4,6 +4,7 @@ import unittest
 
 from testutils import ADMIN_CLIENT
 from testutils import TEARDOWN
+import base
 from library.user import User
 from library.project import Project
 from library.chart import Chart
@@ -49,6 +50,7 @@ class TestProjects(unittest.TestCase):
         user_chart_password = "Aa123456"
         TestProjects.CHART_NAME = 'mariadb'
         TestProjects.VERSION = '4.3.1'
+        base.run_command( ["curl", r"-o", "./tests/apitests/python/mariadb-4.3.1.tgz", "https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz"])
 
         #1. Create a new user(UA);
         TestProjects.user_chart_id, user_chart_name = self.user.create_user(user_password = user_chart_password, **ADMIN_CLIENT)

tests/apitests/python/testutils.py

@@ -22,6 +22,11 @@ ADMIN_CLIENT=dict(endpoint = os.environ.get("HARBOR_HOST_SCHEMA", "https")+ "://
 USER_ROLE=dict(admin=0,normal=1)
 TEARDOWN = os.environ.get('TEARDOWN', 'true').lower() in ('true', 'yes')
 notary_url = os.environ.get('NOTARY_URL', 'https://'+harbor_server+':4443')
+DOCKER_USER = os.environ.get('DOCKER_USER', '')
+DOCKER_PWD = os.environ.get('DOCKER_PWD', '')
+METRIC_URL = os.environ.get('METRIC_URL', 'http://'+harbor_server+':9090')
+BASE_IMAGE = dict(name='busybox', tag='latest')
+BASE_IMAGE_ABS_PATH_NAME = '/' + BASE_IMAGE['name'] + '.tar'
 
 def GetProductApi(username, password, harbor_server= os.environ["HARBOR_HOST"]):
 

tests/ci/api_common_install.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+set -x
 set +e
 sudo rm -fr /data/*
 sudo mkdir -p /data
@@ -9,23 +10,51 @@ set -e
 # prepare cert ...
 sudo sed "s/127.0.0.1/$1/" -i tests/generateCerts.sh
 sudo ./tests/generateCerts.sh
-sudo mkdir -p /etc/docker/certs.d/$1 && sudo cp ./tests/harbor_ca.crt /etc/docker/certs.d/$1/ && rm -rf ~/.docker/ &&  mkdir -p ~/.docker/tls/$1:4443/ && sudo cp ./tests/harbor_ca.crt ~/.docker/tls/$1:4443/
+sudo mkdir -p /etc/docker/certs.d/$1 && sudo cp ./tests/harbor_ca.crt $DIR/../../tests/ca.crt && sudo cp ./tests/harbor_ca.crt /etc/docker/certs.d/$1/ && rm -rf ~/.docker/ &&  mkdir -p ~/.docker/tls/$1:4443/ && sudo cp ./tests/harbor_ca.crt ~/.docker/tls/$1:4443/
+ls -l $DIR/../../tests
 
 sudo ./tests/hostcfg.sh
 
+#---------------Set DNS for docker v20--------------------------#
+# In docker v20, it fixed an issue named  "Wrong resolv.conf    #
+# used on Ubuntu 19", this fix caused DNS solve problem         #
+# in container. So the current work round is read DNS server    #
+# from system and set the value in /etc/docker/daemon.json.     #
+#                                                               #
+# Note: In LDAP pipeline, this setting must be done before      #
+# LDAP prepare phase, since LDAP service is a docker service.   #
+
+ip addr
+dns_ip=$(netplan ip leases eth0 | grep -i dns | awk -F = '{print $2}')
+dns_ip_list=$(echo $dns_ip | tr " " "\n")
+dns_cfg=""
+for ip in $dns_ip_list
+do
+    dns_cfg="$dns_cfg,\"$ip\""
+done
+
+cat /etc/docker/daemon.json
+
+if [ $(cat /etc/docker/daemon.json |grep \"dns\" |wc -l) -eq 0 ];then
+    sudo sed "s/}/,\n   \"dns\": [${dns_cfg:1}]\n}/" -i /etc/docker/daemon.json
+fi
+
+cat /etc/docker/daemon.json
+sudo systemctl daemon-reload
+sudo systemctl restart docker
+sudo systemctl status docker
+#                                                               #
+#---------------------------------------------------------------#
+
+
 if [ "$2" = 'LDAP' ]; then
     cd tests && sudo ./ldapprepare.sh && cd ..
 fi
 
-sudo apt-get update -y && sudo apt-get install -y python3.6
+python --version
-sudo rm /usr/bin/python && sudo ln -s /usr/bin/python3.6 /usr/bin/python
+pip -V
 
-
+#sudo apt-get update && sudo apt-get install -y --no-install-recommends libssl-dev && sudo apt-get autoremove -y && sudo rm -rf /var/lib/apt/lists/*
-# prepare a chart file for API_DB test...
+sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests --upgrade
-sudo curl -o $DIR/../../tests/apitests/python/mariadb-4.3.1.tgz https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz
-
-sudo apt-get update && sudo apt-get install -y --no-install-recommends python-dev openjdk-7-jdk libssl-dev && sudo apt-get autoremove -y && sudo rm -rf /var/lib/apt/lists/*
-sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.0.4 robotframework-httplibrary requests dbbot robotframework-pabot --upgrade
-sudo make swagger_client
 sudo make install GOBUILDIMAGE=golang:1.15.6 COMPILETAG=compile_golangimage CLARITYIMAGE=goharbor/harbor-clarity-ui-builder:1.6.0 NOTARYFLAG=true CLAIRFLAG=true CHARTFLAG=true
 sleep 10

tests/ci/api_run.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-
+set -x
 #source gskey.sh
 
 sudo gsutil version -l
@@ -31,9 +31,11 @@ docker ps
 python --version
 pip -V
 
+E2E_IMAGE="goharbor/harbor-e2e-engine:1.9"
+
 # run db auth api cases
 if [ "$1" = 'DB' ]; then
-    robot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_DB.robot
+    docker run -i --privileged -v $DIR/../../:/drone -v $DIR/../:/ca -w /drone $E2E_IMAGE robot -v DOCKER_USER:${DOCKER_USER} -v DOCKER_PWD:${DOCKER_PWD} -v ip:$2  -v ip1: -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot /drone/tests/robot-cases/Group0-BAT/API_DB.robot
 elif [ "$1" = 'LDAP' ]; then
     # run ldap api cases
     python $DIR/../../tests/configharbor.py -H $IP -u $HARBOR_ADMIN -p $HARBOR_ADMIN_PASSWD -c auth_mode=ldap_auth \
@@ -42,11 +44,14 @@ elif [ "$1" = 'LDAP' ]; then
                                   ldap_search_password=admin \
                                   ldap_base_dn=dc=example,dc=com \
                                   ldap_uid=cn
-    robot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_LDAP.robot
+    docker run -i --privileged -v $DIR/../../:/drone -v $DIR/../:/ca -w /drone $E2E_IMAGE robot -v DOCKER_USER:${DOCKER_USER} -v DOCKER_PWD:${DOCKER_PWD} -v ip:$2  -v ip1: -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot /drone/tests/robot-cases/Group0-BAT/API_LDAP.robot
 else
     rc=999
 fi
 rc=$?
+
+ls -l $DIR/../
+
 ## --------------------------------------------- Upload Harbor CI Logs -------------------------------------------
 timestamp=$(date +%s)
 outfile="integration_logs_$timestamp$TRAVIS_COMMIT.tar.gz"

tests/e2e-image/Dockerfile

@@ -1,28 +1,39 @@
-FROM golang:1.11.2
+FROM ubuntu:18.04
+ENV LANG C.UTF-8
+# V 1.9
 
-RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
+RUN apt-get update && apt-get install -y --no-install-recommends wget curl gnupg2
+RUN apt-get install libseccomp2
+RUN wget  --no-check-certificate -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
 RUN sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
 
-#RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && apt install ./google-chrome-stable_current_amd64.deb
-
 RUN apt-get update && apt-get install -y --no-install-recommends \
     jq \
     bc \
     time \
     gcc \
+    git \
     python-dev \
     libffi-dev \
     libssl-dev \
     sshpass \
     ant \
-	ant-optional \
+    ant-optional \
-    openjdk-7-jdk \
+    xvfb \
+    libxi6 \
+    libgconf-2-4 \
+    openjdk-8-jdk \
     rpcbind \
     nfs-common \
     unzip \
     zip \
     bzip2 \
-	parted \
+    parted \
+    #ip tool
+    #ethtool \
+    iproute2 \
+    #bridge-utils \
+    #iputils-ping \
     # Add docker in docker support
     btrfs-tools \
     e2fsprogs \
@@ -32,7 +43,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     netcat \
     # Add headless chrome support
     google-chrome-stable \
-    Xvfb \
     # Speed up ISO builds with already installed reqs
     yum \
     yum-utils \
@@ -42,34 +52,72 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     xz-utils \
     xorriso \
     sendmail && \
-	# Cleanup
+    # Cleanup
     apt-get autoremove -y && \
     rm -rf /var/lib/apt/lists/*
 
-RUN wget https://bootstrap.pypa.io/get-pip.py && \
+RUN apt-get update && apt-get install -y software-properties-common && \
-    python ./get-pip.py  && \
+    add-apt-repository -y ppa:longsleep/golang-backports
-    pip install pyasn1 google-apitools==0.5.15 gsutil robotframework robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-selenium2library robotframework-pabot --upgrade
+RUN apt-get update && \
+    apt-get install -y  golang-go
+
+RUN apt-get update -y ; apt-get install -y  python3.6
+RUN rm /usr/bin/python ; ln -s /usr/bin/python3.6 /usr/bin/python ; apt-get install -y python3-pip
+RUN python -m pip install --upgrade pip
+
+RUN wget -N http://chromedriver.storage.googleapis.com/2.40/chromedriver_linux64.zip && \
+    unzip chromedriver_linux64.zip && \
+    chmod +x chromedriver && \
+    mv -f chromedriver /usr/local/share/chromedriver && \
+    ln -s /usr/local/share/chromedriver /usr/local/bin/chromedriver && \
+    ln -s /usr/local/share/chromedriver /usr/bin/chromedriver
+
+RUN apt-get update && apt install libnss3-tools && \
+    mkdir -p $HOME/.pki/nssdb && \
+    echo Harbor12345 > password.ca && \
+    certutil -d sql:$HOME/.pki/nssdb -N -f password.ca
+
+#RUN pip install pyasn1 google-apitools==0.5.15 gsutil robotframework==2.9.2 robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-selenium2library robotframework-pabot robotframework-JSONLibrary --upgrade
+
+RUN pip install pyasn1 robotframework==3.0.1 robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-selenium2library robotframework-pabot robotframework-JSONLibrary --upgrade
 
 # Install docker, docker compose
-RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-17.12.0-ce.tgz && \
+ENV DOCKER_VERSION 19.03.12
-    tar --strip-components=1 -xvzf docker-17.12.0-ce.tgz -C /usr/bin &&  \
+RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-$DOCKER_VERSION.tgz && \
-    curl -L https://github.com/docker/compose/releases/download/1.11.2/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose && \
+    tar --strip-components=1 -xvzf docker-$DOCKER_VERSION.tgz -C /usr/bin &&  \
+    curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose && \
     chmod +x /usr/local/bin/docker-compose
 
 RUN wget https://github.com/drone/drone-cli/releases/download/v0.8.3/drone_linux_amd64.tar.gz && tar zxf drone_linux_amd64.tar.gz && \
-    install -t /usr/local/bin drone
+    install -t /usr/local/bin drone && mv drone drone_src
+
+RUN set -x \
+        && groupadd --system dockremap \
+        && adduser --system --ingroup dockremap dockremap \
+        && echo 'dockremap:165536:65536' >> /etc/subuid \
+        && echo 'dockremap:165536:65536' >> /etc/subgid
 
 RUN curl -sSL https://github.com/vmware/govmomi/releases/download/v0.16.0/govc_linux_amd64.gz | gzip -d > /usr/local/bin/govc && \
     chmod +x /usr/local/bin/govc
 
-RUN wget https://get.helm.sh/helm-v2.14.1-linux-386.tar.gz && tar zxvf helm-v2.14.1-linux-386.tar.gz && \
+RUN wget https://get.helm.sh/helm-v2.16.12-linux-amd64.tar.gz && tar zxvf helm-v2.16.12-linux-amd64.tar.gz && \
-    mv linux-386/helm /usr/local/bin/helm && \
+    cp linux-amd64/helm /usr/local/bin/helm && \
-	helm init --client-only && \
+    cp linux-amd64/helm /usr/local/bin/helm2 && \
-	helm plugin install https://github.com/chartmuseum/helm-push
+    helm init --stable-repo-url https://charts.helm.sh/stable --client-only && \
+    helm plugin install https://github.com/chartmuseum/helm-push
 
-RUN wget https://get.helm.sh/helm-v3.0.0-linux-386.tar.gz && tar zxvf helm-v3.0.0-linux-386.tar.gz && \
+RUN wget https://get.helm.sh/helm-v3.3.3-linux-amd64.tar.gz && tar zxvf helm-v3.3.3-linux-amd64.tar.gz && \
-    mv linux-386/helm /usr/local/bin/helm3 && \
+    mv linux-amd64/helm /usr/local/bin/helm3 && \
-	helm3 plugin install https://github.com/chartmuseum/helm-push
+    helm3 plugin install https://github.com/chartmuseum/helm-push
+
+RUN curl -LO https://github.com/deislabs/oras/releases/download/v0.9.0/oras_0.9.0_linux_amd64.tar.gz && \
+    mkdir -p oras-install/  && \
+    tar -zxf oras_0.9.0_*.tar.gz -C oras-install/  && \
+    mv oras-install/oras /usr/local/bin/
+
+RUN wget https://github.com/theupdateframework/notary/releases/download/v0.6.1/notary-Linux-amd64 && \
+    chmod +x notary-Linux-amd64 && \
+    mv notary-Linux-amd64 /usr/local/bin/notary
 
 RUN wget http://nchc.dl.sourceforge.net/sourceforge/tcl/tcl8.4.11-src.tar.gz && \
     tar xfvz tcl8.4.11-src.tar.gz && \
@@ -91,26 +139,20 @@ RUN wget http://nchc.dl.sourceforge.net/sourceforge/tcl/tcl8.4.11-src.tar.gz &&
     cd /ixdba.net/bin && \
     mv  expect /usr/local/bin/expect
 
-# Add docker in docker support
-# version: docker:1.13-dind
-# reference: https://github.com/docker-library/docker/blob/b202ec7e529f5426e2ad7e8c0a8b82cacd406573/1.13/dind/Dockerfile
-#
-# https://github.com/docker/docker/blob/master/project/PACKAGERS.md#runtime-dependencies
 
-# set up subuid/subgid so that "--userns-remap=default" works out-of-the-box
+RUN apt-get install -y sudo uuid-dev
-RUN set -x \
+
-        && groupadd --system dockremap \
-        && adduser --system --ingroup dockremap dockremap \
-        && echo 'dockremap:165536:65536' >> /etc/subuid \
-        && echo 'dockremap:165536:65536' >> /etc/subgid
 
 ENV DIND_COMMIT 3b5fac462d21ca164b3778647420016315289034
 
 RUN wget "https://raw.githubusercontent.com/docker/docker/${DIND_COMMIT}/hack/dind" -O /usr/local/bin/dind \
-        && chmod +x /usr/local/bin/dind
+    && chmod +x /usr/local/bin/dind
+
+COPY busybox.tar /
 
 # This container needs to be run in privileged mode(run with --privileged option) to make it work
 COPY dockerd-entrypoint.sh /usr/local/bin/dockerd-entrypoint.sh
+
 RUN chmod +x /usr/local/bin/dockerd-entrypoint.sh
 
 VOLUME /var/lib/docker

tests/ldapprepare.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -x
 NAME=ldap_server
 docker rm -f $NAME 2>/dev/null
 
@@ -13,6 +14,7 @@ docker run --env LDAP_ORGANISATION="Harbor." \
 sleep 5
 docker cp ldap_test.ldif ldap_server:/
 docker exec ldap_server ldapadd -x -D "cn=admin,dc=example,dc=com" -w admin -f /ldap_test.ldif -ZZ
+docker ps
 
 # failed and retry
 for number in {1..10}

tests/resources/APITest-Util.robot

@@ -14,4 +14,5 @@ Harbor API Test
     Log To Console  ${ip}
     ${rc}  ${output}=  Run And Return Rc And Output  SWAGGER_CLIENT_PATH=${current_dir}/harborclient HARBOR_HOST=${ip} python ${testcase_name}
     Log To Console  ${output}
+    Log   ${output}
     Should Be Equal As Integers  ${rc}  0

tests/resources/Docker-Util.robot

@@ -122,6 +122,8 @@ Prepare Docker Cert
     [Arguments]  ${ip}
     Wait Unitl Command Success  mkdir -p /etc/docker/certs.d/${ip}
     Wait Unitl Command Success  cp harbor_ca.crt /etc/docker/certs.d/${ip}
+    Wait Unitl Command Success  cp harbor_ca.crt /usr/local/share/ca-certificates/
+    Wait Unitl Command Success  update-ca-certificates
 
 Kill Local Docker Daemon
     [Arguments]  ${handle}  ${dockerd-pid}

tests/resources/Nightly-Util.robot

@@ -26,14 +26,14 @@ Nightly Test Setup
     Run Keyword If  '${ip1}' != '${EMPTY}'  Run  rm -rf ./harbor_ca.crt
     Run Keyword  CA setup  ${ip}  ${HARBOR_PASSWORD}
     Run Keyword  Start Docker Daemon Locally
-    Wait Unitl Command Success  docker login -u ${DOCKER_USER} -p ${DOCKER_PWD}
+    Run Keyword If  '${DOCKER_USER}' != '${EMPTY}'  Docker Login  ""  ${DOCKER_USER}  ${DOCKER_PWD}
 
 CA Setup
     [Arguments]  ${ip}  ${HARBOR_PASSWORD}  ${cert}=/ca/ca.crt
     Run  mv ${cert} harbor_ca.crt
     Generate Certificate Authority For Chrome  ${HARBOR_PASSWORD}
     Prepare Docker Cert  ${ip}
-    Prepare Helm Cert
+    Enable Notary Client
 
 Collect Nightly Logs
     [Arguments]  ${ip}  ${SSH_PWD}  ${ip1}==${EMPTY}

tests/robot-cases/Group0-BAT/API_DB.robot

@@ -17,45 +17,86 @@ ${SERVER_API_ENDPOINT}  ${SERVER_URL}/api
 
 *** Test Cases ***
 Test Case - Garbage Collection
+    [tags]  gc
     Harbor API Test  ./tests/apitests/python/test_garbage_collection.py
+
 Test Case - Add Private Project Member and Check User Can See It
+    [tags]  pro_private_member
     Harbor API Test  ./tests/apitests/python/test_add_member_to_private_project.py
+
 Test Case - Delete a Repository of a Certain Project Created by Normal User
+    [tags]  del_repo
     Harbor API Test  ./tests/apitests/python/test_del_repo.py
+
 Test Case - Add a System Global Label to a Certain Tag
+    [tags]  global_lbl
     Harbor API Test  ./tests/apitests/python/test_add_sys_label_to_tag.py
+
 Test Case - Add Replication Rule
+    [tags]  replication_rule
     Harbor API Test  ./tests/apitests/python/test_add_replication_rule.py
+
 Test Case - Edit Project Creation
+    [tags]  pro_creation
     Harbor API Test  ./tests/apitests/python/test_edit_project_creation.py
+
 Test Case - Scan Image
+    [tags]  scan
     Harbor API Test  ./tests/apitests/python/test_scan_image.py
+
 Test Case - Manage Project Member
+    [tags]  pro_member
     Harbor API Test  ./tests/apitests/python/test_manage_project_member.py
+
 Test Case - Project Level Policy Content Trust
+    [tags]  content_trust
     Harbor API Test  ./tests/apitests/python/test_project_level_policy_content_trust.py
+
 Test Case - User View Logs
+    [tags]  logs
     Harbor API Test  ./tests/apitests/python/test_user_view_logs.py
+
 Test Case - Scan All Images
+    [tags]  scan_all
     Harbor API Test  ./tests/apitests/python/test_scan_all_images.py
+
 Test Case - List Helm Charts
+    [tags]  list_helm_charts
     Harbor API Test  ./tests/apitests/python/test_list_helm_charts.py
+
 Test Case - Assign Sys Admin
+    [tags]  sys_admin
     Harbor API Test  ./tests/apitests/python/test_assign_sys_admin.py
+
 Test Case - Retag Image
+    [tags]  retag
     Harbor API Test  ./tests/apitests/python/test_retag.py
+
 Test Case - Robot Account
+    [tags]  robot_account
     Harbor API Test  ./tests/apitests/python/test_robot_account.py
+
 Test Case - Sign A Image
+    [tags]  sign
     Harbor API Test  ./tests/apitests/python/test_sign_image.py
+
 Test Case - Project Quota
+    [tags]  quota
     Harbor API Test  ./tests/apitests/python/test_project_quota.py
+
 Test Case - System Level CVE Whitelist
+    [tags]  sys_cve
     Harbor API Test  ./tests/apitests/python/test_sys_cve_whitelists.py
+
 Test Case - Project Level CVE Whitelist
+    [tags]  pro_cve
     Harbor API Test  ./tests/apitests/python/test_project_level_cve_whitelist.py
+
 Test Case - Tag Retention
+    [tags]  tag_retention
     Harbor API Test  ./tests/apitests/python/test_retention.py
+
 Test Case - Health Check
+    [tags]  health_check
     Harbor API Test  ./tests/apitests/python/test_health_check.py