Merge pull request #14398 from danfengliu/remove-python3.6-in-v1.10.0

[Cherry-pick 1.10.0] Remove deploytment of python3.6 and set dns for docker

tests/apitests/python/library/base.py

@@ -2,6 +2,7 @@
 
 import sys
 import time
+import subprocess
 import swagger_client
 try:
     from urllib import getproxies
@@ -60,6 +61,24 @@ def _get_string_from_unicode(udata):
         result = result + tmp.strip('\n\r\t')
     return result
 
+def run_command(command, expected_error_message = None):
+    print("Command: ", subprocess.list2cmdline(command))
+    try:
+        output = subprocess.check_output(command,
+                                         stderr=subprocess.STDOUT,
+                                         universal_newlines=True)
+    except subprocess.CalledProcessError as e:
+        print("Run command error:", str(e))
+        print("expected_error_message:", expected_error_message)
+        if expected_error_message is not None:
+            if str(e.output).lower().find(expected_error_message.lower()) < 0:
+                raise Exception(r"Error message {} is not as expected {}".format(str(e.output), expected_error_message))
+        else:
+            raise Exception('Error: Exited with error code: %s. Output:%s'% (e.returncode, e.output))
+    else:
+        print("output:", output)
+        return
+
 class Base:
     def __init__(self,
         server = Server(endpoint="http://localhost:8080/api", verify_ssl=False),

tests/apitests/python/library/docker_api.py

@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
 
 import base
+import subprocess
+import json
+from testutils import DOCKER_USER, DOCKER_PWD, BASE_IMAGE, BASE_IMAGE_ABS_PATH_NAME
 
 try:
     import docker
@@ -8,9 +11,86 @@ except ImportError:
     import pip
     pip.main(['install', 'docker'])
     import docker
+
+def docker_info_display():
+    command = ["docker", "info", "-f", "'{{.OSType}}/{{.Architecture}}'"]
+    print("Docker Info: ", command)
+    ret = base.run_command(command)
+    print("Command return: ", ret)
+
+def docker_login_cmd(harbor_host, username, password, cfg_file = "./tests/apitests/python/update_docker_cfg.sh",  enable_manifest = True):
+    if  username == "" or password == "":
+        print("[Warnig]: No docker credential was provided.")
+        return
+    command = ["sudo", "docker", "login", harbor_host, "-u", username, "-p", password]
+    print( "Docker Login Command: ", command)
+    base.run_command(command)
+    if enable_manifest == True:
+        try:
+            ret = subprocess.check_output([cfg_file], shell=False)
+            print("docker login cmd ret:", ret)
+        except subprocess.CalledProcessError as exc:
+            raise Exception("Failed to update docker config, error is {} {}.".format(exc.returncode, exc.output))
+
+def docker_manifest_create(index, manifests):
+    command = ["sudo", "docker","manifest","create", "--amend", index]
+    command.extend(manifests)
+    print( "Docker Manifest Command: ", command)
+    base.run_command(command)
+
+def docker_images_all_list():
+    command = ["sudo", "docker","images","-a"]
+    base.run_command(command)
+
+def docker_load_image(image):
+    command = ["sudo", "docker","load","-i", image]
+    base.run_command(command)
+
+def docker_manifest_push(index):
+    command = ["sudo", "docker","manifest","push",index]
+    print( "Docker Manifest Command: ", command)
+    ret = base.run_command(command)
+    index_sha256=""
+    manifest_list=[]
+    for line in ret.split("\n"):
+        if line[:7] == "sha256:":
+            index_sha256 = line
+        if line.find('Pushed ref') == 0:
+            manifest_list.append(line[-71:])
+    return index_sha256, manifest_list
+
+def docker_manifest_push_to_harbor(index, manifests, harbor_server, username, password, cfg_file = "./tests/apitests/python/update_docker_cfg.sh"):
+    docker_login_cmd(harbor_server, username, password, cfg_file=cfg_file)
+    docker_manifest_create(index, manifests)
+    return docker_manifest_push(index)
+
+def list_repositories(harbor_host, username, password, n = None, last = None):
+    if n is not None and last is not None:
+        command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/_catalog"+"?n=%d"%n+"&last="+last, "--insecure"]
+    elif n is not None:
+            command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/_catalog"+"?n=%d"%n, "--insecure"]
+    else:
+        command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/_catalog", "--insecure"]
+    print( "List Repositories Command: ", command)
+    ret = base.run_command(command)
+    repos = json.loads(ret).get("repositories","")
+    return repos
+
+def list_image_tags(harbor_host, repository, username, password, n = None, last = None):
+    if n is not None and last is not None:
+        command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/"+repository+"/tags/list"+"?n=%d"%n+"&last="+last, "--insecure"]
+    elif n is not None:
+        command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/"+repository+"/tags/list"+"?n=%d"%n, "--insecure"]
+    else:
+        command = ["curl", "-s", "-u", username+":"+password, "https://"+harbor_host+"/v2/"+repository+"/tags/list", "--insecure"]
+    print( "List Image Tags Command: ", command)
+    ret = base.run_command(command)
+    tags = json.loads(ret).get("tags","")
+    return tags
+
 class DockerAPI(object):
     def __init__(self):
-        self.DCLIENT = docker.APIClient(base_url='unix://var/run/docker.sock',version='auto',timeout=10)
+        self.DCLIENT = docker.APIClient(base_url='unix://var/run/docker.sock',version='auto',timeout=30)
         self.DCLIENT2 = docker.from_env()
 
     def docker_login(self, registry, username, password, expected_error_message = None):
@@ -37,12 +117,12 @@ class DockerAPI(object):
         finally:
             if expected_error_message is not None:
                 if str(err_message).lower().find(expected_error_message.lower()) < 0:
-                    raise Exception(r" Failed to catch error [{}] when login image {}, return message: {}".format (expected_error_message, image, err_message))
+                    raise Exception(r" Failed to catch error [{}] when login registry {}, return message: {}".format (expected_error_message, registry, err_message))
                 else:
                     print(r"Docker image login got expected error message:{}".format(expected_error_message))
             else:
                 if str(err_message).lower().find("error".lower()) >= 0:
-                    raise Exception(r" It's was not suppose to catch error when login image {}, return message is [{}]".format (image, err_message))
+                    raise Exception(r" It's was not suppose to catch error when login registry {}, return message is [{}]".format (registry, err_message))
 
     def docker_image_pull(self, image, tag = None, expected_error_message = None):
         ret = ""
@@ -88,6 +168,7 @@ class DockerAPI(object):
     def docker_image_push(self, harbor_registry, tag, expected_error_message = None):
         ret = ""
         err_message = ""
+        docker_images_all_list()
         if expected_error_message is "":
             expected_error_message = None
         try:
@@ -109,16 +190,17 @@ class DockerAPI(object):
             else:
                 if str(err_message).lower().find("error".lower()) >= 0:
                     raise Exception(r" It's was not suppose to catch error when push image {}, return message is [{}]".format (harbor_registry, err_message))
+        docker_images_all_list()
 
     def docker_image_build(self, harbor_registry, tags=None, size=1, expected_error_message = None):
         ret = ""
         err_message = ""
         try:
-            baseimage='busybox:latest'
+            baseimage = BASE_IMAGE['name'] + ":" + BASE_IMAGE['tag']
             if not self.DCLIENT.images(name=baseimage):
-                print( "Docker pull is triggered when building {}".format(harbor_registry))
+                print( "Docker load is triggered when building {}".format(harbor_registry))
-                self.DCLIENT.pull(baseimage)
+                docker_load_image(BASE_IMAGE_ABS_PATH_NAME)
-            c=self.DCLIENT.create_container(image='busybox:latest',
+            c = self.DCLIENT.create_container(image=baseimage,
                 command='dd if=/dev/urandom of=test bs=1M count={}'.format(size))
             self.DCLIENT.start(c)
             self.DCLIENT.wait(c)
@@ -135,10 +217,7 @@ class DockerAPI(object):
                 ret = self.DCLIENT.push(repo)
                 print("docker_image_push ret:", ret)
                 print("build image {} with size {}".format(repo, size))
-                self.DCLIENT.remove_image(repo)
             self.DCLIENT.remove_container(c)
-            #self.DCLIENT.pull(repo)
-            #image = self.DCLIENT2.images.get(repo)
         except Exception as err:
             print( "Docker image build catch exception:", str(err))
             err_message = str(err)

tests/apitests/python/test_list_helm_charts.py

@@ -4,6 +4,7 @@ import unittest
 
 from testutils import ADMIN_CLIENT
 from testutils import TEARDOWN
+import base
 from library.user import User
 from library.project import Project
 from library.chart import Chart
@@ -49,6 +50,7 @@ class TestProjects(unittest.TestCase):
         user_chart_password = "Aa123456"
         TestProjects.CHART_NAME = 'mariadb'
         TestProjects.VERSION = '4.3.1'
+        base.run_command( ["curl", r"-o", "./tests/apitests/python/mariadb-4.3.1.tgz", "https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz"])
 
         #1. Create a new user(UA);
         TestProjects.user_chart_id, user_chart_name = self.user.create_user(user_password = user_chart_password, **ADMIN_CLIENT)

tests/apitests/python/testutils.py

@@ -22,6 +22,11 @@ ADMIN_CLIENT=dict(endpoint = os.environ.get("HARBOR_HOST_SCHEMA", "https")+ "://
 USER_ROLE=dict(admin=0,normal=1)
 TEARDOWN = os.environ.get('TEARDOWN', 'true').lower() in ('true', 'yes')
 notary_url = os.environ.get('NOTARY_URL', 'https://'+harbor_server+':4443')
+DOCKER_USER = os.environ.get('DOCKER_USER', '')
+DOCKER_PWD = os.environ.get('DOCKER_PWD', '')
+METRIC_URL = os.environ.get('METRIC_URL', 'http://'+harbor_server+':9090')
+BASE_IMAGE = dict(name='busybox', tag='latest')
+BASE_IMAGE_ABS_PATH_NAME = '/' + BASE_IMAGE['name'] + '.tar'
 
 def GetProductApi(username, password, harbor_server= os.environ["HARBOR_HOST"]):
 

tests/ci/api_common_install.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+set -x
 set +e
 sudo rm -fr /data/*
 sudo mkdir -p /data
@@ -9,23 +10,51 @@ set -e
 # prepare cert ...
 sudo sed "s/127.0.0.1/$1/" -i tests/generateCerts.sh
 sudo ./tests/generateCerts.sh
-sudo mkdir -p /etc/docker/certs.d/$1 && sudo cp ./tests/harbor_ca.crt /etc/docker/certs.d/$1/ && rm -rf ~/.docker/ &&  mkdir -p ~/.docker/tls/$1:4443/ && sudo cp ./tests/harbor_ca.crt ~/.docker/tls/$1:4443/
+sudo mkdir -p /etc/docker/certs.d/$1 && sudo cp ./tests/harbor_ca.crt $DIR/../../tests/ca.crt && sudo cp ./tests/harbor_ca.crt /etc/docker/certs.d/$1/ && rm -rf ~/.docker/ &&  mkdir -p ~/.docker/tls/$1:4443/ && sudo cp ./tests/harbor_ca.crt ~/.docker/tls/$1:4443/
+ls -l $DIR/../../tests
 
 sudo ./tests/hostcfg.sh
 
+#---------------Set DNS for docker v20--------------------------#
+# In docker v20, it fixed an issue named  "Wrong resolv.conf    #
+# used on Ubuntu 19", this fix caused DNS solve problem         #
+# in container. So the current work round is read DNS server    #
+# from system and set the value in /etc/docker/daemon.json.     #
+#                                                               #
+# Note: In LDAP pipeline, this setting must be done before      #
+# LDAP prepare phase, since LDAP service is a docker service.   #
+
+ip addr
+dns_ip=$(netplan ip leases eth0 | grep -i dns | awk -F = '{print $2}')
+dns_ip_list=$(echo $dns_ip | tr " " "\n")
+dns_cfg=""
+for ip in $dns_ip_list
+do
+    dns_cfg="$dns_cfg,\"$ip\""
+done
+
+cat /etc/docker/daemon.json
+
+if [ $(cat /etc/docker/daemon.json |grep \"dns\" |wc -l) -eq 0 ];then
+    sudo sed "s/}/,\n   \"dns\": [${dns_cfg:1}]\n}/" -i /etc/docker/daemon.json
+fi
+
+cat /etc/docker/daemon.json
+sudo systemctl daemon-reload
+sudo systemctl restart docker
+sudo systemctl status docker
+#                                                               #
+#---------------------------------------------------------------#
+
+
 if [ "$2" = 'LDAP' ]; then
     cd tests && sudo ./ldapprepare.sh && cd ..
 fi
 
-sudo apt-get update -y && sudo apt-get install -y python3.6
+python --version
-sudo rm /usr/bin/python && sudo ln -s /usr/bin/python3.6 /usr/bin/python
+pip -V
 
-
+#sudo apt-get update && sudo apt-get install -y --no-install-recommends libssl-dev && sudo apt-get autoremove -y && sudo rm -rf /var/lib/apt/lists/*
-# prepare a chart file for API_DB test...
+sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests --upgrade
-sudo curl -o $DIR/../../tests/apitests/python/mariadb-4.3.1.tgz https://storage.googleapis.com/harbor-builds/bin/charts/mariadb-4.3.1.tgz
-
-sudo apt-get update && sudo apt-get install -y --no-install-recommends python-dev openjdk-7-jdk libssl-dev && sudo apt-get autoremove -y && sudo rm -rf /var/lib/apt/lists/*
-sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.0.4 robotframework-httplibrary requests dbbot robotframework-pabot --upgrade
-sudo make swagger_client
 sudo make install GOBUILDIMAGE=golang:1.15.6 COMPILETAG=compile_golangimage CLARITYIMAGE=goharbor/harbor-clarity-ui-builder:1.6.0 NOTARYFLAG=true CLAIRFLAG=true CHARTFLAG=true
 sleep 10

tests/ci/api_run.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-
+set -x
 #source gskey.sh
 
 sudo gsutil version -l
@@ -31,9 +31,11 @@ docker ps
 python --version
 pip -V
 
+E2E_IMAGE="goharbor/harbor-e2e-engine:1.9"
+
 # run db auth api cases
 if [ "$1" = 'DB' ]; then
-    robot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_DB.robot
+    docker run -i --privileged -v $DIR/../../:/drone -v $DIR/../:/ca -w /drone $E2E_IMAGE robot -v DOCKER_USER:${DOCKER_USER} -v DOCKER_PWD:${DOCKER_PWD} -v ip:$2  -v ip1: -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot /drone/tests/robot-cases/Group0-BAT/API_DB.robot
 elif [ "$1" = 'LDAP' ]; then
     # run ldap api cases
     python $DIR/../../tests/configharbor.py -H $IP -u $HARBOR_ADMIN -p $HARBOR_ADMIN_PASSWD -c auth_mode=ldap_auth \
@@ -42,11 +44,14 @@ elif [ "$1" = 'LDAP' ]; then
                                   ldap_search_password=admin \
                                   ldap_base_dn=dc=example,dc=com \
                                   ldap_uid=cn
-    robot -v ip:$2 -v HARBOR_PASSWORD:Harbor12345 $DIR/../../tests/robot-cases/Group0-BAT/API_LDAP.robot
+    docker run -i --privileged -v $DIR/../../:/drone -v $DIR/../:/ca -w /drone $E2E_IMAGE robot -v DOCKER_USER:${DOCKER_USER} -v DOCKER_PWD:${DOCKER_PWD} -v ip:$2  -v ip1: -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot /drone/tests/robot-cases/Group0-BAT/API_LDAP.robot
 else
     rc=999
 fi
 rc=$?
+
+ls -l $DIR/../
+
 ## --------------------------------------------- Upload Harbor CI Logs -------------------------------------------
 timestamp=$(date +%s)
 outfile="integration_logs_$timestamp$TRAVIS_COMMIT.tar.gz"

tests/e2e-image/Dockerfile

@@ -1,28 +1,39 @@
-FROM golang:1.11.2
+FROM ubuntu:18.04
+ENV LANG C.UTF-8
+# V 1.9
 
-RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
+RUN apt-get update && apt-get install -y --no-install-recommends wget curl gnupg2
+RUN apt-get install libseccomp2
+RUN wget  --no-check-certificate -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
 RUN sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
 
-#RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && apt install ./google-chrome-stable_current_amd64.deb
-
 RUN apt-get update && apt-get install -y --no-install-recommends \
     jq \
     bc \
     time \
     gcc \
+    git \
     python-dev \
     libffi-dev \
     libssl-dev \
     sshpass \
     ant \
     ant-optional \
-    openjdk-7-jdk \
+    xvfb \
+    libxi6 \
+    libgconf-2-4 \
+    openjdk-8-jdk \
     rpcbind \
     nfs-common \
     unzip \
     zip \
     bzip2 \
     parted \
+    #ip tool
+    #ethtool \
+    iproute2 \
+    #bridge-utils \
+    #iputils-ping \
     # Add docker in docker support
     btrfs-tools \
     e2fsprogs \
@@ -32,7 +43,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     netcat \
     # Add headless chrome support
     google-chrome-stable \
-    Xvfb \
     # Speed up ISO builds with already installed reqs
     yum \
     yum-utils \
@@ -46,31 +56,69 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     apt-get autoremove -y && \
     rm -rf /var/lib/apt/lists/*
 
-RUN wget https://bootstrap.pypa.io/get-pip.py && \
+RUN apt-get update && apt-get install -y software-properties-common && \
-    python ./get-pip.py  && \
+    add-apt-repository -y ppa:longsleep/golang-backports
-    pip install pyasn1 google-apitools==0.5.15 gsutil robotframework robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-selenium2library robotframework-pabot --upgrade
+RUN apt-get update && \
+    apt-get install -y  golang-go
+
+RUN apt-get update -y ; apt-get install -y  python3.6
+RUN rm /usr/bin/python ; ln -s /usr/bin/python3.6 /usr/bin/python ; apt-get install -y python3-pip
+RUN python -m pip install --upgrade pip
+
+RUN wget -N http://chromedriver.storage.googleapis.com/2.40/chromedriver_linux64.zip && \
+    unzip chromedriver_linux64.zip && \
+    chmod +x chromedriver && \
+    mv -f chromedriver /usr/local/share/chromedriver && \
+    ln -s /usr/local/share/chromedriver /usr/local/bin/chromedriver && \
+    ln -s /usr/local/share/chromedriver /usr/bin/chromedriver
+
+RUN apt-get update && apt install libnss3-tools && \
+    mkdir -p $HOME/.pki/nssdb && \
+    echo Harbor12345 > password.ca && \
+    certutil -d sql:$HOME/.pki/nssdb -N -f password.ca
+
+#RUN pip install pyasn1 google-apitools==0.5.15 gsutil robotframework==2.9.2 robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-selenium2library robotframework-pabot robotframework-JSONLibrary --upgrade
+
+RUN pip install pyasn1 robotframework==3.0.1 robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-selenium2library robotframework-pabot robotframework-JSONLibrary --upgrade
 
 # Install docker, docker compose
-RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-17.12.0-ce.tgz && \
+ENV DOCKER_VERSION 19.03.12
-    tar --strip-components=1 -xvzf docker-17.12.0-ce.tgz -C /usr/bin &&  \
+RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-$DOCKER_VERSION.tgz && \
-    curl -L https://github.com/docker/compose/releases/download/1.11.2/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose && \
+    tar --strip-components=1 -xvzf docker-$DOCKER_VERSION.tgz -C /usr/bin &&  \
+    curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose && \
     chmod +x /usr/local/bin/docker-compose
 
 RUN wget https://github.com/drone/drone-cli/releases/download/v0.8.3/drone_linux_amd64.tar.gz && tar zxf drone_linux_amd64.tar.gz && \
-    install -t /usr/local/bin drone
+    install -t /usr/local/bin drone && mv drone drone_src
+
+RUN set -x \
+        && groupadd --system dockremap \
+        && adduser --system --ingroup dockremap dockremap \
+        && echo 'dockremap:165536:65536' >> /etc/subuid \
+        && echo 'dockremap:165536:65536' >> /etc/subgid
 
 RUN curl -sSL https://github.com/vmware/govmomi/releases/download/v0.16.0/govc_linux_amd64.gz | gzip -d > /usr/local/bin/govc && \
     chmod +x /usr/local/bin/govc
 
-RUN wget https://get.helm.sh/helm-v2.14.1-linux-386.tar.gz && tar zxvf helm-v2.14.1-linux-386.tar.gz && \
+RUN wget https://get.helm.sh/helm-v2.16.12-linux-amd64.tar.gz && tar zxvf helm-v2.16.12-linux-amd64.tar.gz && \
-    mv linux-386/helm /usr/local/bin/helm && \
+    cp linux-amd64/helm /usr/local/bin/helm && \
-	helm init --client-only && \
+    cp linux-amd64/helm /usr/local/bin/helm2 && \
+    helm init --stable-repo-url https://charts.helm.sh/stable --client-only && \
     helm plugin install https://github.com/chartmuseum/helm-push
 
-RUN wget https://get.helm.sh/helm-v3.0.0-linux-386.tar.gz && tar zxvf helm-v3.0.0-linux-386.tar.gz && \
+RUN wget https://get.helm.sh/helm-v3.3.3-linux-amd64.tar.gz && tar zxvf helm-v3.3.3-linux-amd64.tar.gz && \
-    mv linux-386/helm /usr/local/bin/helm3 && \
+    mv linux-amd64/helm /usr/local/bin/helm3 && \
     helm3 plugin install https://github.com/chartmuseum/helm-push
 
+RUN curl -LO https://github.com/deislabs/oras/releases/download/v0.9.0/oras_0.9.0_linux_amd64.tar.gz && \
+    mkdir -p oras-install/  && \
+    tar -zxf oras_0.9.0_*.tar.gz -C oras-install/  && \
+    mv oras-install/oras /usr/local/bin/
+
+RUN wget https://github.com/theupdateframework/notary/releases/download/v0.6.1/notary-Linux-amd64 && \
+    chmod +x notary-Linux-amd64 && \
+    mv notary-Linux-amd64 /usr/local/bin/notary
+
 RUN wget http://nchc.dl.sourceforge.net/sourceforge/tcl/tcl8.4.11-src.tar.gz && \
     tar xfvz tcl8.4.11-src.tar.gz && \
     cd tcl8.4.11/unix && \
@@ -91,26 +139,20 @@ RUN wget http://nchc.dl.sourceforge.net/sourceforge/tcl/tcl8.4.11-src.tar.gz &&
     cd /ixdba.net/bin && \
     mv  expect /usr/local/bin/expect
 
-# Add docker in docker support
-# version: docker:1.13-dind
-# reference: https://github.com/docker-library/docker/blob/b202ec7e529f5426e2ad7e8c0a8b82cacd406573/1.13/dind/Dockerfile
-#
-# https://github.com/docker/docker/blob/master/project/PACKAGERS.md#runtime-dependencies
 
-# set up subuid/subgid so that "--userns-remap=default" works out-of-the-box
+RUN apt-get install -y sudo uuid-dev
-RUN set -x \
+
-        && groupadd --system dockremap \
-        && adduser --system --ingroup dockremap dockremap \
-        && echo 'dockremap:165536:65536' >> /etc/subuid \
-        && echo 'dockremap:165536:65536' >> /etc/subgid
 
 ENV DIND_COMMIT 3b5fac462d21ca164b3778647420016315289034
 
 RUN wget "https://raw.githubusercontent.com/docker/docker/${DIND_COMMIT}/hack/dind" -O /usr/local/bin/dind \
     && chmod +x /usr/local/bin/dind
 
+COPY busybox.tar /
+
 # This container needs to be run in privileged mode(run with --privileged option) to make it work
 COPY dockerd-entrypoint.sh /usr/local/bin/dockerd-entrypoint.sh
+
 RUN chmod +x /usr/local/bin/dockerd-entrypoint.sh
 
 VOLUME /var/lib/docker

tests/ldapprepare.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -x
 NAME=ldap_server
 docker rm -f $NAME 2>/dev/null
 
@@ -13,6 +14,7 @@ docker run --env LDAP_ORGANISATION="Harbor." \
 sleep 5
 docker cp ldap_test.ldif ldap_server:/
 docker exec ldap_server ldapadd -x -D "cn=admin,dc=example,dc=com" -w admin -f /ldap_test.ldif -ZZ
+docker ps
 
 # failed and retry
 for number in {1..10}

tests/resources/APITest-Util.robot

@@ -14,4 +14,5 @@ Harbor API Test
     Log To Console  ${ip}
     ${rc}  ${output}=  Run And Return Rc And Output  SWAGGER_CLIENT_PATH=${current_dir}/harborclient HARBOR_HOST=${ip} python ${testcase_name}
     Log To Console  ${output}
+    Log   ${output}
     Should Be Equal As Integers  ${rc}  0

tests/resources/Docker-Util.robot

@@ -122,6 +122,8 @@ Prepare Docker Cert
     [Arguments]  ${ip}
     Wait Unitl Command Success  mkdir -p /etc/docker/certs.d/${ip}
     Wait Unitl Command Success  cp harbor_ca.crt /etc/docker/certs.d/${ip}
+    Wait Unitl Command Success  cp harbor_ca.crt /usr/local/share/ca-certificates/
+    Wait Unitl Command Success  update-ca-certificates
 
 Kill Local Docker Daemon
     [Arguments]  ${handle}  ${dockerd-pid}

tests/resources/Nightly-Util.robot

@@ -26,14 +26,14 @@ Nightly Test Setup
     Run Keyword If  '${ip1}' != '${EMPTY}'  Run  rm -rf ./harbor_ca.crt
     Run Keyword  CA setup  ${ip}  ${HARBOR_PASSWORD}
     Run Keyword  Start Docker Daemon Locally
-    Wait Unitl Command Success  docker login -u ${DOCKER_USER} -p ${DOCKER_PWD}
+    Run Keyword If  '${DOCKER_USER}' != '${EMPTY}'  Docker Login  ""  ${DOCKER_USER}  ${DOCKER_PWD}
 
 CA Setup
     [Arguments]  ${ip}  ${HARBOR_PASSWORD}  ${cert}=/ca/ca.crt
     Run  mv ${cert} harbor_ca.crt
     Generate Certificate Authority For Chrome  ${HARBOR_PASSWORD}
     Prepare Docker Cert  ${ip}
-    Prepare Helm Cert
+    Enable Notary Client
 
 Collect Nightly Logs
     [Arguments]  ${ip}  ${SSH_PWD}  ${ip1}==${EMPTY}

tests/robot-cases/Group0-BAT/API_DB.robot

@@ -17,45 +17,86 @@ ${SERVER_API_ENDPOINT}  ${SERVER_URL}/api
 
 *** Test Cases ***
 Test Case - Garbage Collection
+    [tags]  gc
     Harbor API Test  ./tests/apitests/python/test_garbage_collection.py
+
 Test Case - Add Private Project Member and Check User Can See It
+    [tags]  pro_private_member
     Harbor API Test  ./tests/apitests/python/test_add_member_to_private_project.py
+
 Test Case - Delete a Repository of a Certain Project Created by Normal User
+    [tags]  del_repo
     Harbor API Test  ./tests/apitests/python/test_del_repo.py
+
 Test Case - Add a System Global Label to a Certain Tag
+    [tags]  global_lbl
     Harbor API Test  ./tests/apitests/python/test_add_sys_label_to_tag.py
+
 Test Case - Add Replication Rule
+    [tags]  replication_rule
     Harbor API Test  ./tests/apitests/python/test_add_replication_rule.py
+
 Test Case - Edit Project Creation
+    [tags]  pro_creation
     Harbor API Test  ./tests/apitests/python/test_edit_project_creation.py
+
 Test Case - Scan Image
+    [tags]  scan
     Harbor API Test  ./tests/apitests/python/test_scan_image.py
+
 Test Case - Manage Project Member
+    [tags]  pro_member
     Harbor API Test  ./tests/apitests/python/test_manage_project_member.py
+
 Test Case - Project Level Policy Content Trust
+    [tags]  content_trust
     Harbor API Test  ./tests/apitests/python/test_project_level_policy_content_trust.py
+
 Test Case - User View Logs
+    [tags]  logs
     Harbor API Test  ./tests/apitests/python/test_user_view_logs.py
+
 Test Case - Scan All Images
+    [tags]  scan_all
     Harbor API Test  ./tests/apitests/python/test_scan_all_images.py
+
 Test Case - List Helm Charts
+    [tags]  list_helm_charts
     Harbor API Test  ./tests/apitests/python/test_list_helm_charts.py
+
 Test Case - Assign Sys Admin
+    [tags]  sys_admin
     Harbor API Test  ./tests/apitests/python/test_assign_sys_admin.py
+
 Test Case - Retag Image
+    [tags]  retag
     Harbor API Test  ./tests/apitests/python/test_retag.py
+
 Test Case - Robot Account
+    [tags]  robot_account
     Harbor API Test  ./tests/apitests/python/test_robot_account.py
+
 Test Case - Sign A Image
+    [tags]  sign
     Harbor API Test  ./tests/apitests/python/test_sign_image.py
+
 Test Case - Project Quota
+    [tags]  quota
     Harbor API Test  ./tests/apitests/python/test_project_quota.py
+
 Test Case - System Level CVE Whitelist
+    [tags]  sys_cve
     Harbor API Test  ./tests/apitests/python/test_sys_cve_whitelists.py
+
 Test Case - Project Level CVE Whitelist
+    [tags]  pro_cve
     Harbor API Test  ./tests/apitests/python/test_project_level_cve_whitelist.py
+
 Test Case - Tag Retention
+    [tags]  tag_retention
     Harbor API Test  ./tests/apitests/python/test_retention.py
+
 Test Case - Health Check
+    [tags]  health_check
     Harbor API Test  ./tests/apitests/python/test_health_check.py