Recovery nightly setup

1. Recovery nightly setup;
2. Add retry for replication rule selection.
3. Add retry for CNAB test.
4. Rerun keyword of push CNAB

Signed-off-by: danfengliu <danfengl@vmware.com>

tests/resources/Harbor-Pages/Replication.robot

@@ -167,7 +167,7 @@ Delete Rule
 
 Select Rule
     [Arguments]  ${rule}
-    Retry Element Click  //clr-dg-row[contains(.,'${rule}')]//label
+    Retry Double Keywords When Error  Retry Element Click  //clr-dg-cell[contains(.,'${rule}')]  Retry Wait Element  ${replication_exec_id}
 
 Stop Jobs
     Retry Element Click  ${stop_jobs_button}

tests/resources/Helm-Util.robot

@@ -38,4 +38,4 @@ Helm Chart Push
     [Arguments]  ${ip}  ${user}  ${pwd}  ${chart_file}  ${archive}  ${project}  ${repo_name}  ${verion}
     ${rc}  ${output}=  Run And Return Rc And Output  ./tests/robot-cases/Group0-Util/helm_push_chart.sh ${ip} ${user} ${pwd} ${chart_file} ${archive} ${project} ${repo_name} ${verion}
     Log  ${output}
-    Should Be Equal As Integers  ${rc}  0
+    Should Be Equal As Integers  ${rc}  0

tests/resources/Nightly-Util.robot

@@ -34,6 +34,20 @@ CA Setup
     Prepare Docker Cert  ${ip}
     Prepare Helm Cert
 
+Nightly Test Setup For Nightly
+    [Arguments]  ${ip}  ${HARBOR_PASSWORD}  ${ip1}==${EMPTY}
+    Run Keyword If  '${ip1}' != '${EMPTY}'  CA setup For Nightly  ${ip1}  ${HARBOR_PASSWORD}  /ca/ca1.crt
+    Run Keyword If  '${ip1}' != '${EMPTY}'  Run  rm -rf ./harbor_ca.crt
+    Run Keyword  CA setup For Nightly  ${ip}  ${HARBOR_PASSWORD}
+    Run Keyword  Start Docker Daemon Locally
+
+CA Setup For Nightly
+    [Arguments]  ${ip}  ${HARBOR_PASSWORD}  ${cert}=/ca/ca.crt
+    Run  cp ${cert} harbor_ca.crt
+    Generate Certificate Authority For Chrome  ${HARBOR_PASSWORD}
+    Prepare Docker Cert  ${ip}
+    Prepare Helm Cert
+
 Collect Nightly Logs
     [Arguments]  ${ip}  ${SSH_PWD}  ${ip1}==${EMPTY}
     Run Keyword  Collect Logs  ${ip}  ${SSH_PWD}

tests/robot-cases/Group1-Nightly/Common.robot

@@ -619,7 +619,7 @@ Test Case - Push CNAB Bundle and Display
     Create An New Project And Go Into Project  test${d}
 
     ${target}=  Set Variable  ${ip}/test${d}/cnab${d}:cnab_tag${d}
-    CNAB Push Bundle  ${ip}  user010  Test1@34  ${target}  ./tests/robot-cases/Group0-Util/bundle.json
+    Retry Keyword N Times When Error  5  CNAB Push Bundle  ${ip}  user010  Test1@34  ${target}  ./tests/robot-cases/Group0-Util/bundle.json
 
     Go Into Project  test${d}
     Wait Until Page Contains  test${d}/cnab${d}

tests/robot-cases/Group1-Nightly/Setup_Nightly.robot

@@ -0,0 +1,27 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+*** Settings ***
+Documentation  Harbor BATs
+Resource  ../../resources/Util.robot
+Default Tags  Nightly
+
+*** Test Cases ***
+Test Suites Setup For UI Test
+    Nightly Test Setup For Nightly  ${ip}  ${HARBOR_PASSWORD}  ${ip1}
+    Setup API Test
+
+Test Case - Get Harbor Version
+#Just get harbor version and log it
+    Get Harbor Version

tests/robot-cases/Group3-Upgrade/prepare.py

@@ -403,7 +403,6 @@ class HarborAPI:
                 pass
         open(target, 'wb').write(ca_content.encode('utf-8'))
 
-
 def request(url, method, user = None, userp = None, **kwargs):
     if user is None:
         user = "admin"
@@ -493,3 +492,4 @@ def do_data_creation():
     harborAPI.add_sys_allowlist(data["configuration"]["deployment_security"], version=args.version)
 
 do_data_creation()
+