Add 2.0 & 2.1 upgrade pipeline in nightly

1. Add 2.0 upgrade pipeline in nightly;
2. Add 2.1 upgrade pipeline in nightly.

Signed-off-by: danfengliu <danfengl@vmware.com>

tests/apitests/python/library/docker_api.py

@@ -17,18 +17,18 @@ def docker_info_display():
     ret = base.run_command(command)
     print("Command return: ", ret)
 
-def docker_login_cmd(harbor_host, user, password, enable_manifest = True):
+def docker_login_cmd(harbor_host, user, password, cfg_file = "./tests/apitests/python/update_docker_cfg.sh",  enable_manifest = True):
     command = ["sudo", "docker", "login", harbor_host, "-u", user, "-p", password]
     print( "Docker Login Command: ", command)
     base.run_command(command)
     if enable_manifest == True:
         try:
-            ret = subprocess.check_output(["./tests/apitests/python/update_docker_cfg.sh"], shell=False)
+            ret = subprocess.check_output([cfg_file], shell=False)
         except subprocess.CalledProcessError as exc:
             raise Exception("Failed to update docker config, error is {} {}.".format(exc.returncode, exc.output))
 
 def docker_manifest_create(index, manifests):
-    command = ["sudo", "docker","manifest","create",index]
+    command = ["sudo", "docker","manifest","create", "--amend", index]
     command.extend(manifests)
     print( "Docker Manifest Command: ", command)
     base.run_command(command)
@@ -46,8 +46,8 @@ def docker_manifest_push(index):
             manifest_list.append(line[-71:])
     return index_sha256, manifest_list
 
-def docker_manifest_push_to_harbor(index, manifests, harbor_server, user, password):
+def docker_manifest_push_to_harbor(index, manifests, harbor_server, user, password, cfg_file = "./tests/apitests/python/update_docker_cfg.sh"):
-    docker_login_cmd(harbor_server, user, password)
+    docker_login_cmd(harbor_server, user, password, cfg_file=cfg_file)
     docker_manifest_create(index, manifests)
     return docker_manifest_push(index)
 

tests/resources/Harbor-Pages/Verify.robot

@@ -24,7 +24,7 @@ Verify User
     Close Browser
 
 Verify Project
-    [Arguments]    ${json}
+    [Arguments]    ${json}  ${check_content_trust}=${true}
     Log To Console  "Verify Project..."
     @{project}=  Get Value From Json  ${json}  $.projects.[*].name
     Init Chrome Driver
@@ -32,7 +32,7 @@ Verify Project
     FOR    ${project}    IN    @{project}
         Retry Wait Until Page Contains    ${project}
     END
-    Verify Project Metadata  ${json}
+    Verify Project Metadata  ${json}  ${check_content_trust}
     Close Browser
 
 Verify Image Tag
@@ -43,7 +43,7 @@ Verify Image Tag
     Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
     FOR    ${project}    IN    @{project}
         @{out_has_image}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].has_image
-        ${has_image}  Set Variable If  @{out_has_image}[0] == ${true}  ${true}  ${false}
+        ${has_image}  Set Variable If  ${out_has_image}[0] == ${true}  ${true}  ${false}
         Go Into Project  ${project}  has_image=${has_image}
         @{repo}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})]..repo..name
         Run Keyword If  ${has_image} == ${true}  Loop Image Repo  @{repo}
@@ -52,17 +52,17 @@ Verify Image Tag
     Close Browser
 
 Verify Project Metadata
-    [Arguments]    ${json}
+    [Arguments]    ${json}  ${check_content_trust}
     @{project}=  Get Value From Json  ${json}  $.projects.[*].name
     Init Chrome Driver
     Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
     FOR    ${project}    IN    @{project}
         @{out_has_image}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].has_image
-        ${has_image}  Set Variable If  @{out_has_image}[0] == ${true}  ${true}  ${false}
+        ${has_image}  Set Variable If  ${out_has_image}[0] == ${true}  ${true}  ${false}
         Go Into Project  ${project}  has_image=${has_image}
         Switch To Project Configuration
         Verify Checkbox  ${json}  $.projects[?(@.name=${project})].configuration.public  ${project_config_public_checkbox}
-        Verify Checkbox  ${json}  $.projects[?(@.name=${project})].configuration.enable_content_trust  ${project_config_content_trust_checkbox}
+        Run Keyword If  '${check_content_trust}' == '${true}'  Verify Checkbox  ${json}  $.projects[?(@.name=${project})].configuration.enable_content_trust  ${project_config_content_trust_checkbox}
         Verify Checkbox  ${json}  $.projects[?(@.name=${project})].configuration.auto_scan  ${project_config_scan_images_on_push_checkbox}
         Verify Checkbox  ${json}  $.projects[?(@.name=${project})].configuration.prevent_vul  ${project_config_prevent_vulnerable_images_from_running_checkbox}
         ${ret}    Get Selected List Value    ${project_config_severity_select}
@@ -94,7 +94,7 @@ Verify Member Exist
     Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
     FOR    ${project}    IN    @{project}
         @{out_has_image}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].has_image
-        ${has_image}  Set Variable If  @{out_has_image}[0] == ${true}  ${true}  ${false}
+        ${has_image}  Set Variable If  ${out_has_image}[0] == ${true}  ${true}  ${false}
         Go Into Project  ${project}  has_image=${has_image}
         Switch To Member
         @{members}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].member..name
@@ -111,7 +111,7 @@ Verify Webhook
     Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
     FOR    ${project}    IN    @{project}
         @{out_has_image}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].has_image
-        ${has_image}  Set Variable If  @{out_has_image}[0] == ${true}  ${true}  ${false}
+        ${has_image}  Set Variable If  ${out_has_image}[0] == ${true}  ${true}  ${false}
         Go Into Project  ${project}  has_image=${has_image}
         Switch To Project Webhooks
         ${enabled}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].webhook.enabled
@@ -130,6 +130,37 @@ Verify Webhook
     END
     Close Browser
 
+Verify Webhook For 2.0
+    [Arguments]    ${json}
+    Log To Console  "Verify Webhook..."
+    @{project}=  Get Value From Json  ${json}  $.projects.[*].name
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    FOR    ${project}    IN    @{project}
+        @{out_has_image}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].has_image
+        ${has_image}  Set Variable If  ${out_has_image}[0] == ${true}  ${true}  ${false}
+        Go Into Project  ${project}  has_image=${has_image}
+        Switch To Project Webhooks
+        ${enabled}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].webhook.enabled
+        ${enable_count}  Get Element Count  xpath=//span[contains(.,'Enabled')]
+        ${disable_count}  Get Element Count  xpath=//span[contains(.,'Disabled')]
+        Log To Console  '${enabled}[0]'
+        Log To Console  '${true}'
+        Run Keyword If  '${enabled}[0]' == '${true}'  Page Should Contain  Enabled
+        ...  ELSE  Page Should Contain  Disabled
+        ${address}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].webhook.address
+        ${name}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].webhook.name
+        ${notify_type}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].webhook.notify_type
+        Log To Console  '${address}[0]'
+        Log To Console  '${name}[0]'
+        Log To Console  '${notify_type}[0]'
+        Page Should Contain  ${address}[0]
+        Page Should Contain  ${name}[0]
+        Page Should Contain  ${notify_type}[0]
+        Navigate To Projects
+    END
+    Close Browser
+
 Verify Tag Retention Rule
     [Arguments]    ${json}
     Log To Console  "Verify Tag Retention Rule..."
@@ -137,6 +168,8 @@ Verify Tag Retention Rule
     Init Chrome Driver
     Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
     FOR    ${project}    IN    @{project}
+        ${tag_retention_rule}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].tag_retention_rule
+        Run Keyword If  ${tag_retention_rule}[0] == ${null}  Continue For Loop
         ${out_has_image}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].has_image
         ${has_image}  Set Variable If  ${out_has_image}[0] == ${true}  ${true}  ${false}
         Go Into Project  ${project}  has_image=${has_image}
@@ -163,7 +196,7 @@ Verify Tag Immutability Rule
     Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
     FOR    ${project}    IN    @{project}
         @{out_has_image}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].has_image
-        ${has_image}  Set Variable If  @{out_has_image}[0] == ${true}  ${true}  ${false}
+        ${has_image}  Set Variable If  ${out_has_image}[0] == ${true}  ${true}  ${false}
         Go Into Project  ${project}  has_image=${has_image}
         Switch To Tag Immutability
         @{repo_decoration}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].tag_immutability_rule.repo_decoration
@@ -193,7 +226,7 @@ Verify Robot Account Exist
     Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
     FOR    ${project}    IN    @{project}
         @{out_has_image}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].has_image
-        ${has_image}  Set Variable If  @{out_has_image}[0] == ${true}  ${true}  ${false}
+        ${has_image}  Set Variable If  ${out_has_image}[0] == ${true}  ${true}  ${false}
         Go Into Project  ${project}  has_image=${has_image}
         Switch To Project Robot Account
         @{robot_accounts}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].robot_account..name
@@ -241,7 +274,7 @@ Verify Project Label
     Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
     FOR    ${project}    IN    @{project}
         @{out_has_image}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].has_image
-        ${has_image}  Set Variable If  @{out_has_image}[0] == ${true}  ${true}  ${false}
+        ${has_image}  Set Variable If  ${out_has_image}[0] == ${true}  ${true}  ${false}
         Go Into Project  ${project}  has_image=${has_image}
         Switch To Project Label
         @{projectlabel}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})]..labels..name
@@ -304,31 +337,6 @@ Verify Replicationrule
     END
     Close Browser
 
-Verify Project Setting
-    [Arguments]    ${json}
-    Log To Console  "Verify Project Setting..."
-    @{projects}=  Get Value From Json  ${json}  $.projects.[*].name
-    FOR    ${project}    IN    @{Projects}
-        ${public}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].accesslevel
-        ${contenttrust}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})]..enable_content_trust
-        ${preventrunning}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})]..prevent_vulnerable_images_from_running
-        ${scanonpush}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})]..automatically_scan_images_on_push
-        Init Chrome Driver
-        Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
-        @{out_has_image}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].has_image
-        ${has_image}  Set Variable If  @{out_has_image}[0] == ${true}  ${true}  ${false}
-        Go Into Project  ${project}  has_image=${has_image}
-        Goto Project Config
-        Run Keyword If  ${public} == "public"  Checkbox Should Be Checked  //clr-checkbox-wrapper[@name='public']//label
-        Run Keyword If  ${contenttrust} == "true"  Checkbox Should Be Checked  //clr-checkbox-wrapper[@name='content-trust']//label
-        Run Keyword If  ${contenttrust} == "false"  Checkbox Should Not Be Checked  //clr-checkbox-wrapper[@name='content-trust']//label
-        Run Keyword If  ${preventrunning} == "true"  Checkbox Should Be Checked  //*[@id='prevent-vulenrability-image']//clr-checkbox-wrapper//label
-        Run Keyword If  ${preventrunning} == "false"  Checkbox Should Not Be Checked    //*[@id='prevent-vulenrability-image']//clr-checkbox-wrapper//label
-        Run Keyword If  ${scanonpush} == "true"  Checkbox Should Be Checked  //clr-checkbox-wrapper[@id='scan-image-on-push-wrapper']//input
-        Run Keyword If  ${scanonpush} == "true"  Checkbox Should Not Be Checked  //clr-checkbox-wrapper[@id='scan-image-on-push-wrapper']//input
-       Close Browser
-    END
-
 Verify Interrogation Services
     [Arguments]    ${json}
     Log To Console  "Verify Interrogation Services..."
@@ -380,7 +388,7 @@ Verify Project-level Allowlist
     Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
     FOR    ${project}    IN    @{project}
         @{out_has_image}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].has_image
-        ${has_image}  Set Variable If  @{out_has_image}[0] == ${true}  ${true}  ${false}
+        ${has_image}  Set Variable If  ${out_has_image}[0] == ${true}  ${true}  ${false}
         Go Into Project  ${project}  has_image=${has_image}
         Switch To Project Configuration
         @{is_reuse_sys_cve_allowlist}=    Get Value From Json    ${json}    $.projects[?(@.name=${project})].configuration.reuse_sys_cve_allowlist
@@ -423,3 +431,83 @@ Verify Trivy Is Default Scanner
     Switch To Scanners Page
     Should Display The Default Trivy Scanner
     Close Browser
+
+Verify Artifact Index
+    [Arguments]    ${json}
+    Log To Console  "Verify Artifact Index..."
+    # Only the 1st project has manifest image, so use index 0 of projects for verification.
+    @{project}=  Get Value From Json  ${json}  $.projects.[0].name
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    FOR    ${project}    IN    @{project}
+        ${name}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].artifact_index.name
+        ${tag}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].artifact_index.tag
+        Go Into Project  ${project}  has_image=${true}
+        Go Into Repo  ${project}/${name}[0]
+        Go Into Index And Contain Artifacts  ${tag}[0]  limit=2
+        Pull image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  ${project}  ${name}[0]:${tag}[0]
+        Navigate To Projects
+    END
+    Close Browser
+
+Loop Repo
+    [Arguments]  ${project}  @{repos}
+    FOR    ${repo}    IN    @{repos}
+        Navigate To Projects
+        Go Into Project  ${project}  has_image=${true}
+        Go Into Repo  ${project}/${repo}[0][cache_image_namespace]/${repo}[0][cache_image]
+        Pull image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  ${project}  ${repo}[0][cache_image_namespace]/${repo}[0][cache_image]:${repo}[0][tag]
+    END
+
+Verify Proxy Cache Image Existence
+    [Arguments]    ${json}
+    Log To Console  "Verify Proxy Cache Image Existence..."
+    # Only the 3rd project has cached image, so use index 2 of projects for verification.
+    @{project}=  Get Value From Json  ${json}  $.projects.[2].name
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    FOR    ${project}    IN    @{project}
+        @{repo}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].repo
+        Loop Repo  ${project}  @{repo}
+    END
+    Close Browser
+
+Verify Distributions
+    [Arguments]    ${json}
+    Log To Console  "Verify Distributions..."
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    @{distribution_names}=  Get Value From Json  ${json}  $.distributions..name
+    Switch To Distribution
+    FOR    ${name}    IN    @{distribution_names}
+        ${endpoint}=  Get Value From Json  ${json}  $.distributions[?(@.name=${name})].endpoint
+        ${vendor}=  Get Value From Json  ${json}  $.distributions[?(@.name=${name})].vendor
+        ${auth_mode}=  Get Value From Json  ${json}  $.distributions[?(@.name=${name})].auth_mode
+        Retry Wait Until Page Contains Element  //div[@class='datagrid-scrolling-cells' and contains(.,'${name}') and contains(.,'${endpoint}[0]') and contains(.,'${vendor}[0]') and contains(.,'${auth_mode}[0]')]
+    END
+
+Verify P2P Preheat Policy
+    [Arguments]    ${json}
+    Log To Console  "P2P Preheat Policy..."
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Navigate To Projects
+    @{project}=  Get Value From Json  ${json}  $.projects.[*].name
+    FOR    ${project}    IN    @{project}
+        @{p2p_preheat_policys}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].p2p_preheat_policy
+        @{policy_names}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].p2p_preheat_policy..name
+        @{out_has_image}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].has_image
+        ${has_image}  Set Variable If  ${out_has_image}[0] == ${true}  ${true}  ${false}
+        Run Keyword If  ${p2p_preheat_policys}[0] == ${null}  Continue For Loop
+        Go Into Project  ${project}  has_image=${has_image}
+        Switch To P2P Preheat
+        Loop P2P Preheat Policys  ${json}  ${project}  @{policy_names}
+    END
+    Close Browser
+
+Loop P2P Preheat Policys
+    [Arguments]  ${json}  ${project}  @{policy_names}
+    FOR    ${policy}    IN    @{policy_names}
+        ${provider_name}=  Get Value From Json  ${json}  $.projects[?(@.name=${project})].p2p_preheat_policy[?(@.name=${policy})].provider_name
+        Retry Wait Until Page Contains Element   //div[@class='datagrid-scrolling-cells' and contains(.,'${policy}') and contains(.,'${provider_name}[0]')]
+    END

tests/robot-cases/Group1-Nightly/Setup.robot

@@ -19,6 +19,7 @@ Default Tags  Nightly
 
 *** Test Cases ***
 Test Suites Setup
+    [Tags]  setup
     Nightly Test Setup  ${ip}  ${HARBOR_PASSWORD}  ${ip1}
     Setup API Test
 

tests/robot-cases/Group3-Upgrade/data.json

@@ -1,380 +1,547 @@
 {
-        "users":[
+   "users":[
-           {
+      {
-              "name":"user1"
+         "name":"user1"
-           },
+      },
-           {
+      {
-              "name":"user2"
+         "name":"user2"
-           },
+      },
-           {
+      {
-              "name":"user3"
+         "name":"user3"
-           },
+      },
-           {
+      {
-              "name":"user4"
+         "name":"user4"
-           },
+      },
-           {
+      {
-              "name":"user5"
+         "name":"user5"
-           }
+      }
-        ],
+   ],
-        "admin":[
+   "admin":[
-           {
+      {
-              "name":"user1",
+         "name":"user1",
-              "has_admin":true
+         "has_admin":true
-           },
+      },
-           {
+      {
-              "name":"user2",
+         "name":"user2",
-              "has_admin":true
+         "has_admin":true
-           }
+      }
-        ],
+   ],
-        "endpoint":[
+   "endpoint":[
-           {
+      {
-              "url":"http://url",
+         "url":"http://url",
-              "name":"endpoint1",
+         "name":"endpoint1",
-              "user":"admin",
+         "user":"admin",
-              "pass":"Harbor12345",
+         "pass":"Harbor12345",
-              "insecure":true,
+         "insecure":true,
-              "type":"harbor"
+         "type":"harbor"
-           },
+      },
-           {
+      {
-               "url":"https://hub.docker.com",
+         "url":"https://hub.docker.com",
-               "name":"endpoint2",
+         "name":"endpoint_for_proxy_cache",
-               "user":"",
+         "user":"",
-               "pass":"",
+         "pass":"",
-               "insecure":false,
+         "insecure":false,
-               "type":"docker-hub"
+         "type":"docker-hub"
-            }
+      }
-        ],
+   ],
-        "replicationrule":[
+   "replicationrule":[
-           {
+      {
-              "project":"project1",
+         "project":"project1",
-              "endpoint":"endpoint1",
+         "endpoint":"endpoint1",
-              "trigger":"Manual",
+         "trigger":"Manual",
-              "rulename":"rulename",
+         "rulename":"rulename",
-              "is_src_registry":false,
+         "is_src_registry":false,
-              "dest_namespace":"rule1-namespace",
+         "dest_namespace":"rule1-namespace",
-              "trigger_type":"event_based",
+         "trigger_type":"event_based",
-              "cron":"",
+         "cron":"",
-              "deletion":true,
+         "deletion":true,
-              "enabled":true,
+         "enabled":true,
-              "override":true,
+         "override":true,
-              "name_filters":"namefilter1",
+         "name_filters":"namefilter1",
-              "tag_filters":"tagfilter1"
+         "tag_filters":"tagfilter1"
-           },
+      },
-           {
+      {
-              "project":"library",
+         "project":"library",
-              "endpoint":"endpoint2",
+         "endpoint":"endpoint_for_proxy_cache",
-              "trigger":"Manual",
+         "trigger":"Manual",
-              "rulename":"rulelibrary",
+         "rulename":"rulelibrary",
-              "endpoint_type":"harbor",
+         "endpoint_type":"harbor",
-              "is_src_registry":true,
+         "is_src_registry":true,
-              "dest_namespace":"rule2-namespace",
+         "dest_namespace":"rule2-namespace",
-              "trigger_type":"scheduled",
+         "trigger_type":"scheduled",
-              "cron":"6 7 8 * * *",
+         "cron":"6 7 8 * * *",
-              "deletion":false,
+         "deletion":false,
-              "enabled":true,
+         "enabled":true,
-              "override":true,
+         "override":true,
-              "name_filters":"namefilter2",
+         "name_filters":"namefilter2",
-              "tag_filters":"tagfilter2"
+         "tag_filters":"tagfilter2"
+      }
+   ],
+   "interrogation_services":{
+      "cron":"0 0 0 6 3 0"
+   },
+   "configuration":{
+      "authmode":"Database",
+      "projectcreation":"everyone",
+      "selfreg":false,
+      "emailsetting":{
+         "emailserver":"emailserver",
+         "emailport":"2220",
+         "emailuser":"emailuser",
+         "emailfrom":"emailfrom",
+         "emailssl":false,
+         "verifycert":false
+      },
+      "token":"40",
+      "robot_token":"31",
+      "readonly":false,
+      "scanall":{
+         "type":"none",
+         "param":{
+            "daily":0
+         }
+      },
+      "deployment_security":{
+         "cve":[
+            {
+               "id":"CVE-2019-12904"
+               },
+            {
+               "id":"CVE-2011-3389"
+            },
+            {
+               "id":"CVE-2018-12886"
+            },
+            {
+               "id":"CVE-2019-3844"
             }
          ],
-         "interrogation_services":{
+         "never_expires":false,
-           "cron":"0 0 0 6 3 0"
+         "expires_at":"2572969600"
+      },
+      "webhooks_enabled":true,
+      "syslabel":[
+         {
+            "name":"label1"
+         },
+         {
+            "name":"label2"
+         },
+         {
+            "name":"label3"
+         },
+         {
+            "name":"label4"
+         },
+         {
+            "name":"label5"
+         },
+         {
+            "name":"label6"
+         }
+      ]
+   },
+   "distributions":[
+      {
+         "name":"distribution124",
+         "endpoint":"http://6.3.3.6",
+         "enabled":true,
+         "vendor":"kraken",
+         "auth_mode":"NONE",
+         "insecure":null
+      },
+      {
+         "name":"distribution134",
+         "endpoint":"http://6.3.6.vio",
+         "enabled":true,
+         "vendor":"dragonfly",
+         "auth_mode":"NONE",
+         "insecure":true
+      }
+   ],
+   "projects":[
+      {
+         "name":"project1194",
+         "registry_name":null,
+         "has_image":true,
+         "accesslevel":"public",
+         "repocounts":2,
+         "repo":[
+            {
+               "name":"busybox",
+               "tag":"latest",
+               "signed":"false"
+            },
+            {
+               "name":"alpine",
+               "tag":"latest",
+               "signed":"true"
+            }
+         ],
+         "artifact_index":{
+            "name":"ci_test_index_232",
+            "tag":"test_tag_989"
+         },
+         "member":[
+            {
+               "name":"user1",
+               "role":1
+            },
+            {
+               "name":"user2",
+               "role":2
+            },
+            {
+               "name":"user3",
+               "role":3
+            }
+         ],
+         "p2p_preheat_policy":null,
+         "count_limit":1234,
+         "storage_limit":53687091200,
+         "storage_limit_for_verify":50,
+         "storage_unit_for_verify":"GB",
+         "replications":{
+            "rulename":"ruleproject1",
+            "endpointname":"endpoint1",
+            "trigger":"Manual"
+         },
+         "labels":[
+            {
+               "name":"proj1label1"
+            },
+            {
+               "name":"proj1label2"
+            },
+            {
+               "name":"proj1label3"
+            }
+         ],
+         "logs":[
+            {
+               "name":"admin",
+               "operation":"create"
+            }
+         ],
+         "robot_account":[
+            {
+               "name":"robot0",
+               "access": [
+                  {
+                  "action": "pull"
+                  },
+                  {
+                  "action": "push"
+                  }
+               ]
+            }
+         ],
+         "tag_retention_rule":{
+            "repository_patten":"*photon*",
+            "tag_decoration":"*v1.*",
+            "latestPushedK":666,
+            "latestPushedK_verify":"666",
+            "cron":"0 0 0 1 8 0"
+         },
+         "tag_immutability_rule":{
+            "repo_decoration":"repoMatches",
+            "tag_decoration":"excludes",
+            "repo_pattern":"*alpine*",
+            "tag_pattern":"*v1.2*"
+         },
+         "webhook":{
+            "address":"https://1.1.1.1",
+            "skip_cert_verify":true,
+            "auth_header":"aaa",
+            "enabled":true,
+            "notify_type":"http",
+            "name":"webhook18"
          },
          "configuration":{
-           "authmode":"Database",
+            "public":"false",
-           "projectcreation":"everyone",
+            "enable_content_trust":"false",
-           "selfreg":false,
+            "auto_scan":"false",
-           "emailsetting":{
+            "prevent_vul":"false",
-              "emailserver":"emailserver",
+            "severity":"medium",
-              "emailport":"2220",
+            "reuse_sys_cve_allowlist":"true",
-              "emailuser":"emailuser",
+            "deployment_security":{
-              "emailfrom":"emailfrom",
+               "cve":[]
-              "emailssl":false,
+            }
-              "verifycert":false
+         }
-           },
+      },
-           "token":"40",
+      {
-           "robot_token":"31",
+         "name":"project2194",
-           "readonly":false,
+         "registry_name":null,
-           "scanall":{
+         "has_image":false,
-              "type":"none",
+         "accesslevel":"public",
-              "param":{
+         "repocounts":2,
-                 "daily":0
+         "repo":[
-              }
+            {
-           },
+               "name":"busybox",
-           "deployment_security":{
+               "tag":"latest",
+               "signed":"false"
+            },
+            {
+               "name":"alpine",
+               "tag":"latest",
+               "signed":"true"
+            }
+         ],
+         "member":[
+            {
+               "name":"user1",
+               "role":1
+            },
+            {
+               "name":"user2",
+               "role":2
+            },
+            {
+               "name":"user3",
+               "role":3
+            }
+         ],
+         "p2p_preheat_policy":[
+            {
+               "provider_name":"distribution124",
+               "name":"p2p_preheat_policy_1",
+               "filters":"[{\"type\":\"repository\",\"value\":\"cent*\"},{\"type\":\"tag\",\"value\":\"v1*\"}]",
+               "trigger":"{\"type\":\"manual\",\"trigger_setting\":{\"cron\":\"\"}}",
+               "project_id":36,
+               "enabled":true
+            },
+            {
+               "provider_name":"distribution134",
+               "name":"p2p_preheat_policy_2",
+               "filters":"[{\"type\":\"repository\",\"value\":\"re*\"},{\"type\":\"tag\",\"value\":\"v2*\"}]",
+               "trigger":"{\"type\":\"manual\",\"trigger_setting\":{\"cron\":\"\"}}",
+               "project_id":36,
+               "enabled":true
+            }
+         ],
+         "count_limit":-1,
+         "storage_limit":32985348833280,
+         "storage_limit_for_verify":30,
+         "storage_unit_for_verify":"TB",
+         "replications":{
+            "rulename":"rulename1",
+            "endpointname":"endpoint1",
+            "trigger":"Manual"
+         },
+         "labels":[
+            {
+               "name":"proj2label1"
+            },
+            {
+               "name":"proj2label2"
+            },
+            {
+               "name":"proj2label3"
+            }
+         ],
+         "logs":[
+            {
+               "name":"admin",
+               "operation":"create"
+            }
+         ],
+         "robot_account":[
+            {
+               "name":"robot1",
+               "access": [
+                  {
+                     "action": "pull"
+                  },
+                  {
+                     "action": "push"
+                  }
+                  ]
+            },
+            {
+               "name":"robot2",
+               "access": [
+                  {
+                     "action": "pull"
+                  }
+                  ]
+            },
+            {
+               "name":"robot3",
+               "access": [
+                  {
+                     "action": "push"
+                  }
+                  ]
+            }
+         ],
+         "tag_retention_rule":{
+            "repository_patten":"*centos*",
+            "tag_decoration":"*latest",
+            "latestPushedK":999,
+            "latestPushedK_verify":"999",
+            "cron":"0 0 0 1 6 0"
+         },
+         "tag_immutability_rule":{
+            "repo_decoration":"repoExcludes",
+            "tag_decoration":"matches",
+            "repo_pattern":"*buxybox*",
+            "tag_pattern":"*v3*"
+         },
+         "webhook":{
+            "address":"https://2.2.2.2",
+            "skip_cert_verify":false,
+            "auth_header":"bbb",
+            "enabled":true,
+            "notify_type":"slack",
+            "name":"webhook28"
+         },
+         "configuration":{
+            "public":"true",
+            "enable_content_trust":"true",
+            "auto_scan":"true",
+            "prevent_vul":"true",
+            "severity":"high",
+            "reuse_sys_cve_allowlist":"false",
+            "deployment_security":{
                "cve":[
                   {
-                     "id":"CVE-2019-12904"
+                     "id":"CVE-2019-111"
                      },
                   {
-                     "id":"CVE-2011-3389"
+                     "id":"CVE-2011-2222"
                   },
                   {
-                     "id":"CVE-2018-12886"
+                     "id":"CVE-2018-33333"
                   },
                   {
-                     "id":"CVE-2019-3844"
+                     "id":"CVE-2019-44444"
                   }
                ],
-              "never_expires":false,
+               "never_expires":false,
-              "expires_at":"2572969600"
+               "expires_at":"2572969600"
-           },
+            }
-           "webhooks_enabled":true,
+         }
-           "syslabel":[
+      },
-              {
+      {
-                 "name":"label1"
+         "name":"project_proxy_3194",
-              },
+         "registry_name":"endpoint_for_proxy_cache",
-              {
+         "p2p_preheat_policy":null,
-                 "name":"label2"
+         "has_image":true,
-              },
+         "accesslevel":"public",
-              {
+         "repocounts":2,
-                 "name":"label3"
+         "repo":[
-              },
+            {
-              {
+               "cache_image_namespace":"library",
-                 "name":"label4"
+               "cache_image":"busybox",
-              },
+               "tag":"latest",
-              {
+               "signed":"false"
-                 "name":"label5"
+            }
-              },
+         ],
-              {
+         "member":[
-                 "name":"label6"
+            {
-              }
+               "name":"user1",
-           ]
+               "role":1
-        },
+            },
-        "projects":[
+            {
-           {
+               "name":"user2",
-              "name":"project1",
+               "role":2
-              "has_image":true,
+            },
-              "accesslevel":"public",
+            {
-              "repocounts":2,
+               "name":"user3",
-              "repo":[
+               "role":3
-                 {
+            }
-                    "name":"busybox",
+         ],
-                    "tag":"latest",
+         "count_limit":-1,
-                    "signed":"false"
+         "storage_limit":32985348833280,
-                 },
+         "storage_limit_for_verify":30,
-                 {
+         "storage_unit_for_verify":"TB",
-                    "name":"alpine",
+         "labels":[
-                    "tag":"latest",
+            {
-                    "signed":"true"
+               "name":"proj2label1"
-                 }
+            },
-              ],
+            {
-              "member":[
+               "name":"proj2label2"
-                 {
+            },
-                    "name":"user1",
+            {
-                    "role":1
+               "name":"proj2label3"
-                 },
+            }
-                 {
+         ],
-                    "name":"user2",
+         "logs":[
-                    "role":2
+            {
-                 },
+               "name":"admin",
-                 {
+               "operation":"create"
-                    "name":"user3",
+            }
-                    "role":3
+         ],
-                 }
+         "robot_account":[
-              ],
+            {
-              "count_limit":1234,
+               "name":"robot1",
-              "storage_limit":53687091200,
+               "access": [
-              "storage_limit_for_verify":50,
-              "storage_unit_for_verify":"GB",
-              "replications":{
-                 "rulename":"ruleproject1",
-                 "endpointname":"endpoint1",
-                 "trigger":"Manual"
-              },
-              "labels":[
-                 {
-                    "name":"proj1label1"
-                 },
-                 {
-                    "name":"proj1label2"
-                 },
-                 {
-                    "name":"proj1label3"
-                 }
-              ],
-              "logs":[
-                 {
-                    "name":"admin",
-                    "operation":"create"
-                 }
-              ],
-              "robot_account":[
                   {
-                     "name":"robot0",
+                     "action": "pull"
-                     "access": [
-                        {
-                        "action": "pull"
-                        },
-                        {
-                        "action": "push"
-                        }
-                     ]
-                  }
-               ],
-               "tag_retention_rule":{
-                  "repository_patten":"*photon*",
-                  "tag_decoration":"*v1.*",
-                  "latestPushedK":666,
-                  "latestPushedK_verify":"666",
-                  "cron":"0 0 0 1 8 0"
-               },
-               "tag_immutability_rule":{
-                  "repo_decoration":"repoMatches",
-                  "tag_decoration":"excludes",
-                  "repo_pattern":"*alpine*",
-                  "tag_pattern":"*v1.2*"
-               },
-               "webhook":{
-                  "address":"https://1.1.1.1",
-                  "skip_cert_verify":true,
-                  "auth_header":"aaa",
-                  "enabled":true
-               },
-              "configuration":{
-                 "public":"true",
-                 "enable_content_trust":"true",
-                 "auto_scan":"true",
-                 "prevent_vul":"true",
-                 "severity":"high",
-                 "reuse_sys_cve_allowlist":"false",
-                 "deployment_security":{
-                  "cve":[
-                     {
-                        "id":"CVE-2019-111"
-                        },
-                     {
-                        "id":"CVE-2011-2222"
-                     },
-                     {
-                        "id":"CVE-2018-33333"
-                     },
-                     {
-                        "id":"CVE-2019-44444"
-                     }
-                  ],
-                 "never_expires":false,
-                 "expires_at":"2572969600"
-               }
-              }
-           },
-           {
-              "name":"project2",
-              "has_image":false,
-              "accesslevel":"public",
-              "repocounts":2,
-              "repo":[
-                 {
-                    "name":"busybox",
-                    "tag":"latest",
-                    "signed":"false"
-                 },
-                 {
-                    "name":"alpine",
-                    "tag":"latest",
-                    "signed":"true"
-                 }
-              ],
-              "member":[
-                 {
-                    "name":"user1",
-                    "role":1
-                 },
-                 {
-                    "name":"user2",
-                    "role":2
-                 },
-                 {
-                    "name":"user3",
-                    "role":3
-                 }
-              ],
-              "count_limit":-1,
-              "storage_limit":32985348833280,
-              "storage_limit_for_verify":30,
-              "storage_unit_for_verify":"TB",
-              "replications":{
-                 "rulename":"rulename1",
-                 "endpointname":"endpoint1",
-                 "trigger":"Manual"
-              },
-              "labels":[
-                 {
-                    "name":"proj2label1"
-                 },
-                 {
-                    "name":"proj2label2"
-                 },
-                 {
-                    "name":"proj2label3"
-                 }
-              ],
-              "logs":[
-                 {
-                    "name":"admin",
-                    "operation":"create"
-                 }
-              ],
-              "robot_account":[
-                  {
-                     "name":"robot1",
-                     "access": [
-                        {
-                          "action": "pull"
-                        },
-                        {
-                          "action": "push"
-                        }
-                      ]
                   },
                   {
-                     "name":"robot2",
+                     "action": "push"
-                     "access": [
-                        {
-                          "action": "pull"
-                        }
-                      ]
-                  },
-                  {
-                     "name":"robot3",
-                     "access": [
-                        {
-                          "action": "push"
-                        }
-                      ]
                   }
-               ],
-               "tag_retention_rule":{
-                  "repository_patten":"*centos*",
-                  "tag_decoration":"*latest",
-                  "latestPushedK":999,
-                  "latestPushedK_verify":"999",
-                  "cron":"0 0 0 1 6 0"
-               },
-               "tag_immutability_rule":{
-                  "repo_decoration":"repoExcludes",
-                  "tag_decoration":"matches",
-                  "repo_pattern":"*buxybox*",
-                  "tag_pattern":"*v3*"
-               },
-               "webhook":{
-                  "address":"https://2.2.2.2",
-                  "skip_cert_verify":false,
-                  "auth_header":"bbb",
-                  "enabled":false
-               },
-              "configuration":{
-                 "public":"false",
-                 "enable_content_trust":"false",
-                 "auto_scan":"false",
-                 "prevent_vul":"true",
-                 "severity":"medium",
-                 "reuse_sys_cve_allowlist":"true",
-                 "deployment_security":{
-                  "cve":[
                   ]
-               }
+            },
-              }
+            {
-           }
+               "name":"robot2",
-        ]
+               "access": [
-     }
+                  {
+                     "action": "pull"
+                  }
+                  ]
+            },
+            {
+               "name":"robot3",
+               "access": [
+                  {
+                     "action": "push"
+                  }
+                  ]
+            }
+         ],
+         "tag_retention_rule":null,
+         "tag_immutability_rule":{
+            "repo_decoration":"repoExcludes",
+            "tag_decoration":"matches",
+            "repo_pattern":"*logstash*",
+            "tag_pattern":"*v2*"
+         },
+         "webhook":{
+            "address":"https://3.3.3.3",
+            "skip_cert_verify":false,
+            "auth_header":"bbb",
+            "enabled":true,
+            "notify_type":"slack",
+            "name":"webhook28"
+         },
+         "configuration":{
+            "public":"true",
+            "enable_content_trust":"true",
+            "auto_scan":"true",
+            "prevent_vul":"true",
+            "severity":"high",
+            "reuse_sys_cve_allowlist":"false",
+            "deployment_security":{
+               "cve":[
+                  {
+                     "id":"CVE-2019-111"
+                     },
+                  {
+                     "id":"CVE-2011-2222"
+                  },
+                  {
+                     "id":"CVE-2018-33333"
+                  },
+                  {
+                     "id":"CVE-2019-44444"
+                  }
+               ],
+               "never_expires":false,
+               "expires_at":"2572969600"
+            }
+         }
+      }
+   ]
+}

tests/robot-cases/Group3-Upgrade/feature_map.json

@@ -23,6 +23,18 @@
         {
             "branch":2,
             "version":"1.10"
+        },
+        {
+            "branch":2,
+            "version":"2.0"
+        },
+        {
+            "branch":3,
+            "version":"2.1"
+        },
+        {
+            "branch":3,
+            "version":"2.2"
         }
     ],
     "add_member":[
@@ -75,6 +87,18 @@
         {
             "branch":2,
             "version":"1.10"
+        },
+        {
+            "branch":3,
+            "version":"2.0"
+        },
+        {
+            "branch":3,
+            "version":"2.1"
+        },
+        {
+            "branch":3,
+            "version":"2.2"
         }
     ],
     "add_endpoint":[
@@ -137,6 +161,18 @@
         {
             "branch":1,
             "version":"1.10"
+        },
+        {
+            "branch":1,
+            "version":"2.0"
+        },
+        {
+            "branch":2,
+            "version":"2.1"
+        },
+        {
+            "branch":2,
+            "version":"2.2"
         }
     ],
     "update_project_setting_allowlist":[
@@ -147,6 +183,18 @@
         {
             "branch":1,
             "version":"1.10"
+        },
+        {
+            "branch":1,
+            "version":"2.0"
+        },
+        {
+            "branch":2,
+            "version":"2.1"
+        },
+        {
+            "branch":2,
+            "version":"2.2"
         }
     ],
     "add_project_robot_account":[
@@ -187,6 +235,18 @@
         {
             "branch":1,
             "version":"1.10"
+        },
+        {
+            "branch":2,
+            "version":"2.0"
+        },
+        {
+            "branch":2,
+            "version":"2.1"
+        },
+        {
+            "branch":2,
+            "version":"2.2"
         }
     ],
     "update_interrogation_services":[
@@ -197,6 +257,128 @@
         {
             "branch":1,
             "version":"1.10"
+        },
+        {
+            "branch":1,
+            "version":"2.0"
+        },
+        {
+            "branch":1,
+            "version":"2.1"
+        },
+        {
+            "branch":1,
+            "version":"2.2"
+        }
+    ],
+    "push_artifact_index":[
+        {
+            "branch":1,
+            "version":"2.0"
+        },
+        {
+            "branch":1,
+            "version":"2.1"
+        },
+        {
+            "branch":1,
+            "version":"2.2"
+        }
+    ],
+    "add_distribution":[
+        {
+            "branch":1,
+            "version":"2.1"
+        },
+        {
+            "branch":1,
+            "version":"2.2"
+        }
+    ],
+    "add_p2p_preheat_policy":[
+        {
+            "branch":1,
+            "version":"2.1"
+        },
+        {
+            "branch":1,
+            "version":"2.2"
+        }
+    ],
+    "get_ca":[
+        {
+            "branch":1,
+            "version":"1.4"
+        },
+        {
+            "branch":1,
+            "version":"1.6"
+        },
+        {
+            "branch":1,
+            "version":"1.7"
+        },
+        {
+            "branch":1,
+            "version":"1.8"
+        },
+        {
+            "branch":1,
+            "version":"1.9"
+        },
+        {
+            "branch":1,
+            "version":"1.10"
+        },
+        {
+            "branch":2,
+            "version":"2.0"
+        },
+        {
+            "branch":2,
+            "version":"2.1"
+        },
+        {
+            "branch":2,
+            "version":"2.2"
+        }
+    ],
+    "set_url":[
+        {
+            "branch":1,
+            "version":"1.4"
+        },
+        {
+            "branch":1,
+            "version":"1.6"
+        },
+        {
+            "branch":1,
+            "version":"1.7"
+        },
+        {
+            "branch":1,
+            "version":"1.8"
+        },
+        {
+            "branch":1,
+            "version":"1.9"
+        },
+        {
+            "branch":1,
+            "version":"1.10"
+        },
+        {
+            "branch":2,
+            "version":"2.0"
+        },
+        {
+            "branch":2,
+            "version":"2.1"
+        },
+        {
+            "branch":2,
+            "version":"2.2"
         }
     ]
  }

tests/robot-cases/Group3-Upgrade/prepare.py

@@ -1,8 +1,10 @@
 import os
 import sys
 import json
+import time
 import argparse
 import requests
+import urllib
 from functools import wraps
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
 requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
@@ -10,8 +12,16 @@ requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 parser = argparse.ArgumentParser(description='The script to generate data for harbor v1.4.0')
 parser.add_argument('--endpoint', '-e', dest='endpoint', required=True, help='The endpoint to harbor')
 parser.add_argument('--version', '-v', dest='version', required=False, help='The version to harbor')
+parser.add_argument('--libpath', '-l', dest='libpath', required=False, help='e2e library')
 args = parser.parse_args()
 
+from os import path
+sys.path.append(args.libpath)
+sys.path.append(args.libpath + "/library")
+from library.docker_api import docker_manifest_push_to_harbor
+from library.repository import push_image_to_project
+from library.repository import Repository
+
 url = "https://"+args.endpoint+"/api/"
 endpoint_url = "https://"+args.endpoint
 print(url)
@@ -48,9 +58,27 @@ class HarborAPI:
     @get_feature_branch
     def create_project(self, project, **kwargs):
         if kwargs["branch"] == 1:
-            body=dict(body={"project_name": ""+project["name"]+"", "metadata": {"public": "true"}})
+            body=dict(body={"project_name": project["name"], "metadata": {"public": "true"}})
         elif kwargs["branch"] == 2:
-            body=dict(body={"project_name": ""+project["name"]+"", "metadata": {"public": "true"},"count_limit":project["count_limit"],"storage_limit":project["storage_limit"]})
+            body=dict(body={"project_name": project["name"], "metadata": {"public": "true"},"count_limit":project["count_limit"],"storage_limit":project["storage_limit"]})
+        elif kwargs["branch"] == 3:
+            if project["registry_name"] is not None:
+                r = request(url+"registries?name="+project["registry_name"]+"", 'get')
+                registry_id = int(str(r.json()[0]['id']))
+            else:
+                registry_id=None
+            body=dict(body={"project_name": project["name"], "registry_id":registry_id, "metadata": {"public": "true"},"storage_limit":project["storage_limit"]})
+            request(url+"projects", 'post', **body)
+            if project["registry_name"] is not None:
+                USER_ADMIN=dict(endpoint = "https://"+args.endpoint+"/api/v2.0" , username = "admin", password = "Harbor12345")
+                repo = Repository()
+                for _repo in project["repo"]:
+                    pull_image(args.endpoint+"/"+ project["name"]+"/"+_repo["cache_image_namespace"]+"/"+_repo["cache_image"])
+                    time.sleep(180)
+                    repo_name = urllib.parse.quote(_repo["cache_image_namespace"]+"/"+_repo["cache_image"],'utf-8')
+                    repo_data = repo.get_repository(project["name"], repo_name, **USER_ADMIN)
+                    print("=========repo_data:",repo_data)
+            return
         else:
             raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
         request(url+"projects", 'post', **body)
@@ -69,6 +97,8 @@ class HarborAPI:
             body=dict(body={"has_admin_role": 1})
         elif kwargs["branch"] == 2:
             body=dict(body={"has_admin_role": True})
+        elif kwargs["branch"] == 3:
+            body=dict(body={"sysadmin_flag": True, "user_id":int(userid)})
         else:
             raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
         request(url+"users/"+userid+"/sysadmin", 'put', **body)
@@ -87,6 +117,38 @@ class HarborAPI:
         body=dict(body=payload)
         request(url+"projects/"+projectid+"/members", 'post', **body)
 
+    @get_feature_branch
+    def add_p2p_preheat_policy(self, project, **kwargs):
+        r = request(url+"projects?name="+project["name"]+"", 'get')
+        projectid = int(str(r.json()[0]['project_id']))
+        if kwargs["branch"] == 1:
+            if project["p2p_preheat_policy"] is not None:
+                instances = request(url+"p2p/preheat/instances", 'get')
+                if len(instances.json()) == 0:
+                    raise Exception(r"Please add p2p preheat instances first.")
+                for instance in instances.json():
+                    print("instance:", instance)
+                    for policy in project["p2p_preheat_policy"]:
+                        instance_str = [str(item) for item in instances]
+                        if policy["provider_name"] in ''.join(instance_str):
+                            print("policy:", policy)
+                            if instance['name'] == policy["provider_name"]:
+                                payload = {
+                                    "provider_id":int(instance['id']),
+                                    "name":policy["name"],
+                                    "filters":policy["filters"],
+                                    "trigger":policy["trigger"],
+                                    "project_id":projectid,
+                                    "enabled":policy["enabled"]
+                                }
+                                body=dict(body=payload)
+                                print(body)
+                                request(url+"projects/"+project["name"]+"/preheat/policies", 'post', **body)
+                        else:
+                            raise Exception(r"Please verify if distribution {} has beed created.".format(policy["provider_name"]))
+        else:
+            raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
+
     @get_feature_branch
     def add_endpoint(self, endpointurl, endpointname, username, password, insecure, registry_type, **kwargs):
         if kwargs["branch"] == 1:
@@ -109,7 +171,7 @@ class HarborAPI:
             }
             body=dict(body=payload)
             print(body)
-            request(url+"/registries", 'post', **body)
+            request(url+"registries", 'post', **body)
         else:
             raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
 
@@ -164,6 +226,13 @@ class HarborAPI:
                     cve_id_str = cve_id_str + ","
             body=dict(body=json.loads(r'{"items":['+cve_id_str+r'],"expires_at":'+cve_id_list["expires_at"]+'}'))
             request(url+"system/CVEWhitelist", 'put', **body)
+        elif kwargs["branch"] == 2:
+            for index, cve_id in enumerate(cve_id_list["cve"]):
+                cve_id_str = cve_id_str + '{"cve_id":"' +cve_id["id"] + '"}'
+                if index != len(cve_id_list["cve"]) - 1:
+                    cve_id_str = cve_id_str + ","
+            body=dict(body=json.loads(r'{"items":['+cve_id_str+r'],"expires_at":'+cve_id_list["expires_at"]+'}'))
+            request(url+"system/CVEAllowlist", 'put', **body)
         else:
             raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
 
@@ -185,6 +254,19 @@ class HarborAPI:
             print(payload)
             body=dict(body=json.loads(payload))
             request(url+"projects/"+projectid+"", 'put', **body)
+        elif kwargs["branch"] == 2:
+            for index, cve_id in enumerate(cve_id_list["cve"]):
+                cve_id_str = cve_id_str + '{"cve_id":"' +cve_id["id"] + '"}'
+                if index != len(cve_id_list["cve"]) - 1:
+                    cve_id_str = cve_id_str + ","
+            print(cve_id_str)
+            if reuse_sys_cve_whitelist == "true":
+                payload = r'{"metadata":{"reuse_sys_cve_allowlist":"true"}}'
+            else:
+                payload = r'{"metadata":{"reuse_sys_cve_allowlist":"false"},"cve_whitelist":{"project_id":'+projectid+',"items":['+cve_id_str+r'],"expires_at":'+cve_id_list["expires_at"]+'}}'
+            print(payload)
+            body=dict(body=json.loads(payload))
+            request(url+"projects/"+projectid+"", 'put', **body)
         else:
             raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
 
@@ -262,6 +344,8 @@ class HarborAPI:
 
     @get_feature_branch
     def add_tag_retention_rule(self, project, tag_retention_rule, **kwargs):
+        if tag_retention_rule is None:
+            print(r"No tag retention rule to be populated for project {}.".format(project))
         r = request(url+"projects?name="+project+"", 'get')
         projectid = str(r.json()[0]['project_id'])
         if kwargs["branch"] == 1:
@@ -308,7 +392,8 @@ class HarborAPI:
             }
             print(payload)
             body=dict(body=payload)
-            request(url+"retentions", 'post', **body)
+            action = "post"
+            request(url+"retentions", action, **body)
         else:
             raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, kwargs["branch"]))
 
@@ -357,7 +442,7 @@ class HarborAPI:
             payload = {
                 "targets":[
                     {
-                        "type":"http",
+                        "type":webhook["notify_type"],
                         "address":webhook["address"],
                         "skip_cert_verify":webhook["skip_cert_verify"],
                         "auth_header":webhook["auth_header"]
@@ -378,6 +463,35 @@ class HarborAPI:
             print(payload)
             body=dict(body=payload)
             request(url+"projects/"+projectid+"/webhook/policies", 'post', **body)
+        elif kwargs["branch"] == 2:
+            payload = {
+                "targets":[
+                    {
+                        "type":webhook["notify_type"],
+                        "address":webhook["address"],
+                        "skip_cert_verify":webhook["skip_cert_verify"],
+                        "auth_header":webhook["auth_header"]
+                    }
+                ],
+                "event_types":[
+                    "DELETE_ARTIFACT",
+                    "PULL_ARTIFACT",
+                    "PUSH_ARTIFACT",
+                    "DELETE_CHART",
+                    "DOWNLOAD_CHART",
+                    "UPLOAD_CHART",
+                    "QUOTA_EXCEED",
+                    "QUOTA_WARNING",
+					"REPLICATION",
+					"SCANNING_FAILED",
+					"SCANNING_COMPLETED"
+                ],
+                "enabled":webhook["enabled"],
+                "name":webhook["name"]
+            }
+            print(payload)
+            body=dict(body=payload)
+            request(url+"projects/"+projectid+"/webhook/policies", 'post', **body)
         else:
             raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, kwargs["branch"]))
 
@@ -387,8 +501,29 @@ class HarborAPI:
         body=dict(body=payload)
         request(url+"repositories/"+reponame+"", 'put', **body)
 
-    def get_ca(self, target='/harbor/ca/ca.crt'):
+    @get_feature_branch
-        url = "https://" + args.endpoint + "/api/systeminfo/getcert"
+    def add_distribution(self, distribution, **kwargs):
+        if kwargs["branch"] == 1:
+            payload = {
+                "name":distribution["name"],
+                "endpoint":distribution["endpoint"],
+                "enabled":distribution["enabled"],
+                "vendor":distribution["vendor"],
+                "auth_mode":distribution["auth_mode"],
+                "insecure":distribution["insecure"]
+            }
+            print(payload)
+            body=dict(body=payload)
+            request(url+"p2p/preheat/instances", 'post', **body)
+        else:
+            raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, kwargs["branch"]))
+
+    @get_feature_branch
+    def get_ca(self, target='/harbor/ca/ca.crt', **kwargs):
+        if kwargs["branch"] == 1:
+            url = "https://" + args.endpoint + "/api/systeminfo/getcert"
+        elif kwargs["branch"] == 2:
+            url = "https://" + args.endpoint + "/api/v2.0/systeminfo/getcert"
         resp = request(url, 'get')
         try:
             ca_content = json.loads(resp.text)
@@ -403,6 +538,17 @@ class HarborAPI:
                 pass
         open(target, 'wb').write(ca_content.encode('utf-8'))
 
+    @get_feature_branch
+    def push_artifact_index(self, project, name, tag, **kwargs):
+        image_a = "alpine"
+        image_b = "busybox"
+        repo_name_a, tag_a = push_image_to_project(project, args.endpoint, 'admin', 'Harbor12345', image_a, "latest")
+        repo_name_b, tag_b = push_image_to_project(project, args.endpoint, 'admin', 'Harbor12345', image_b, "latest")
+
+        #4. Push an index(IA) to Harbor by docker manifest CLI successfully;
+        manifests = [args.endpoint+"/"+repo_name_a+":"+tag_a, args.endpoint+"/"+repo_name_b+":"+tag_b]
+        index = args.endpoint+"/"+project+"/"+name+":"+tag
+        docker_manifest_push_to_harbor(index, manifests, args.endpoint, 'admin', 'Harbor12345', cfg_file = args.libpath + "/update_docker_cfg.sh")
 
 def request(url, method, user = None, userp = None, **kwargs):
     if user is None:
@@ -426,19 +572,29 @@ with open("data.json") as f:
 
 def pull_image(*image):
     for i in image:
+        print("docker pulling image: ", i)
         os.system("docker pull "+i)
 
 def push_image(image, project):
     os.system("docker tag "+image+" "+args.endpoint+"/"+project+"/"+image)
     os.system("docker login "+args.endpoint+" -u admin"+" -p Harbor12345")
-    os.system("docker push "+args.endpoint+"/"+project+"/"+image)
+    os.system("docker push "+args.endpoint+"/"+project+"/library/"+image)
 
 def push_signed_image(image, project, tag):
     os.system("./sign_image.sh" + " " + args.endpoint + " " + project + " " + image + " " + tag)
 
+@get_feature_branch
+def set_url(**kwargs):
+    global url
+    if kwargs["branch"] == 1:
+        url = "https://"+args.endpoint+"/api/"
+    elif kwargs["branch"] == 2:
+        url = "https://"+args.endpoint+"/api/v2.0/"
+
 def do_data_creation():
     harborAPI = HarborAPI()
-    harborAPI.get_ca()
+    set_url(version=args.version)
+    harborAPI.get_ca(version=args.version)
 
     for user in data["users"]:
         harborAPI.create_user(user["name"])
@@ -446,23 +602,31 @@ def do_data_creation():
     for user in data["admin"]:
         harborAPI.set_user_admin(user["name"], version=args.version)
 
+    # Make sure to create endpoint first, it's for proxy cache project creation.
+    for endpoint in data["endpoint"]:
+        harborAPI.add_endpoint(endpoint["url"], endpoint["name"], endpoint["user"], endpoint["pass"], endpoint["insecure"], endpoint["type"], version=args.version)
+
+    for distribution in data["distributions"]:
+        harborAPI.add_distribution(distribution, version=args.version)
+
     for project in data["projects"]:
         harborAPI.create_project(project, version=args.version)
         for member in project["member"]:
             harborAPI.add_member(project["name"], member["name"], member["role"], version=args.version)
         for robot_account in project["robot_account"]:
             harborAPI.add_project_robot_account(project["name"], robot_account, version=args.version)
+        harborAPI.add_p2p_preheat_policy(project, version=args.version)
         harborAPI.add_webhook(project["name"], project["webhook"], version=args.version)
-        harborAPI.add_tag_retention_rule(project["name"], project["tag_retention_rule"], version=args.version)
+        if project["tag_retention_rule"] in not None:
+            harborAPI.add_tag_retention_rule(project["name"], project["tag_retention_rule"], version=args.version)
         harborAPI.add_tag_immutability_rule(project["name"], project["tag_immutability_rule"], version=args.version)
+        time.sleep(30)
 
+    harborAPI.push_artifact_index(data["projects"][0]["name"], data["projects"][0]["artifact_index"]["name"], data["projects"][0]["artifact_index"]["tag"], version=args.version)
     pull_image("busybox", "redis", "haproxy", "alpine", "httpd:2")
     push_image("busybox", data["projects"][0]["name"])
     push_signed_image("alpine", data["projects"][0]["name"], "latest")
 
-    for endpoint in data["endpoint"]:
-        harborAPI.add_endpoint(endpoint["url"], endpoint["name"], endpoint["user"], endpoint["pass"], endpoint["insecure"], endpoint["type"], version=args.version)
-
     for replicationrule in data["replicationrule"]:
         harborAPI.add_replication_rule(replicationrule, version=args.version)
 

tests/robot-cases/Group3-Upgrade/run.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+IP=$1
+HARBOR_VERSION=$2
+
+robot -v ip:$IP  -v ip1: -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot
+cd /drone/tests/robot-cases/Group3-Upgrade
+python ./prepare.py -e $IP -v $HARBOR_VERSION -l /drone/tests/apitests/python/

tests/robot-cases/Group3-Upgrade/verify.robot

@@ -18,7 +18,6 @@ Test Case - Upgrade Verify
     Run Keyword  Verify User System Admin Role  ${data}
     Run Keyword  Verify Endpoint  ${data}
     Run Keyword  Verify Replicationrule  ${data}
-    Run Keyword  Verify Project Setting  ${data}
     Run Keyword  Verify System Setting  ${data}
     Run Keyword  Verify Image Tag  ${data}
 
@@ -35,7 +34,6 @@ Test Case - Upgrade Verify
     Run Keyword  Verify User System Admin Role  ${data}
     Run Keyword  Verify Endpoint  ${data}
     Run Keyword  Verify Replicationrule  ${data}
-    Run Keyword  Verify Project Setting  ${data}
     Run Keyword  Verify Interrogation Services  ${data}
     Run Keyword  Verify System Setting  ${data}
     Run Keyword  Verify System Setting Allowlist  ${data}
@@ -56,9 +54,53 @@ Test Case - Upgrade Verify
     Run Keyword  Verify User System Admin Role  ${data}
     Run Keyword  Verify Endpoint  ${data}
     Run Keyword  Verify Replicationrule  ${data}
-    Run Keyword  Verify Project Setting  ${data}
     Run Keyword  Verify Interrogation Services  ${data}
     Run Keyword  Verify System Setting  ${data}
     Run Keyword  Verify System Setting Allowlist  ${data}
     Run Keyword  Verify Image Tag  ${data}
     Run Keyword  Verify Clair Is Default Scanner
+
+Test Case - Upgrade Verify
+    [Tags]  2.0-latest
+    ${data}=  Load Json From File  ${CURDIR}${/}data.json
+    Run Keyword  Verify User  ${data}
+    Run Keyword  Verify Project  ${data}
+    Run Keyword  Verify Member Exist  ${data}
+    Run Keyword  Verify Robot Account Exist  ${data}
+    Run Keyword  Verify Project-level Allowlist  ${data}
+    Run Keyword  Verify Webhook For 2.0  ${data}
+    Run Keyword  Verify Tag Retention Rule  ${data}
+    Run Keyword  Verify Tag Immutability Rule  ${data}
+    Run Keyword  Verify User System Admin Role  ${data}
+    Run Keyword  Verify Endpoint  ${data}
+    Run Keyword  Verify Replicationrule  ${data}
+    Run Keyword  Verify Interrogation Services  ${data}
+    Run Keyword  Verify System Setting  ${data}
+    Run Keyword  Verify System Setting Allowlist  ${data}
+    Run Keyword  Verify Image Tag  ${data}
+    Run Keyword  Verify Trivy Is Default Scanner
+    Run Keyword  Verify Artifact Index  ${data}
+
+Test Case - Upgrade Verify
+    [Tags]  2.1-latest
+    ${data}=  Load Json From File  ${CURDIR}${/}data.json
+    Run Keyword  Verify User  ${data}
+    Run Keyword  Verify Project  ${data}  check_content_trust=${false}
+    Run Keyword  Verify Member Exist  ${data}
+    Run Keyword  Verify Robot Account Exist  ${data}
+    Run Keyword  Verify Project-level Allowlist  ${data}
+    Run Keyword  Verify Webhook For 2.0  ${data}
+    Run Keyword  Verify Tag Retention Rule  ${data}
+    Run Keyword  Verify Tag Immutability Rule  ${data}
+    Run Keyword  Verify User System Admin Role  ${data}
+    Run Keyword  Verify Endpoint  ${data}
+    Run Keyword  Verify Replicationrule  ${data}
+    Run Keyword  Verify Interrogation Services  ${data}
+    Run Keyword  Verify System Setting  ${data}
+    Run Keyword  Verify System Setting Allowlist  ${data}
+    Run Keyword  Verify Image Tag  ${data}
+    Run Keyword  Verify Trivy Is Default Scanner
+    Run Keyword  Verify Artifact Index  ${data}
+    Run Keyword  Verify Proxy Cache Image Existence  ${data}
+    Run Keyword  Verify Distributions  ${data}
+    Run Keyword  Verify P2P Preheat Policy  ${data}