From e4894017cd4a66574c7b28c342f0cbc45abdec60 Mon Sep 17 00:00:00 2001 From: danfengliu Date: Tue, 22 Sep 2020 09:27:05 +0000 Subject: [PATCH] Fix typo in upgrade prepare scritp Signed-off-by: danfengliu --- tests/resources/Harbor-Pages/Verify.robot | 87 +- tests/robot-cases/Group3-Upgrade/data.json | 6 +- .../Group3-Upgrade/feature_map.json | 950 +++++++++--------- tests/robot-cases/Group3-Upgrade/prepare.py | 73 +- tests/robot-cases/Group3-Upgrade/verify.robot | 222 ++-- 5 files changed, 716 insertions(+), 622 deletions(-) diff --git a/tests/resources/Harbor-Pages/Verify.robot b/tests/resources/Harbor-Pages/Verify.robot index 8bcc8ccdd..eea6e9208 100644 --- a/tests/resources/Harbor-Pages/Verify.robot +++ b/tests/resources/Harbor-Pages/Verify.robot @@ -24,39 +24,29 @@ Verify User Close Browser Verify Project - [Arguments] ${json} ${check_content_trust}=${true} + [Arguments] ${json} ${verify_registry_name}=${false} Log To Console "Verify Project..." @{project}= Get Value From Json ${json} $.projects.[*].name Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} FOR ${project} IN @{project} + @{registry_name}= Get Value From Json ${json} $.projects[?(@.name=${project})].registry_name + Run Keyword If '${registry_name}[0]' != '${null}' and '${verify_registry_name}' == '${false}' Continue For Loop Retry Wait Until Page Contains ${project} END - Verify Project Metadata ${json} ${check_content_trust} Close Browser -Verify Image Tag - [Arguments] ${json} - Log To Console "Verify Image Tag..." - @{project}= Get Value From Json ${json} $.projects.[*].name - Init Chrome Driver - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - FOR ${project} IN @{project} - @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image - ${has_image} Set Variable If ${out_has_image}[0] == ${true} ${true} ${false} - Go Into Project ${project} has_image=${has_image} - @{repo}= Get Value From Json ${json} $.projects[?(@.name=${project})]..repo..name - Run Keyword If ${has_image} == ${true} Loop Image Repo @{repo} - Navigate To Projects - END - Close Browser Verify Project Metadata - [Arguments] ${json} ${check_content_trust} + # check_content_trust has been removed from Harbor since v2.0 + # verify_registry_name is for proxy cache project, this feature developed since 2.1 + [Arguments] ${json} ${check_content_trust}=${true} ${verify_registry_name}=${false} @{project}= Get Value From Json ${json} $.projects.[*].name Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} FOR ${project} IN @{project} + @{registry_name}= Get Value From Json ${json} $.projects[?(@.name=${project})].registry_name + Run Keyword If '${registry_name}[0]' != '${null}' and '${verify_registry_name}' == '${false}' Continue For Loop @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image ${has_image} Set Variable If ${out_has_image}[0] == ${true} ${true} ${false} Go Into Project ${project} has_image=${has_image} @@ -72,6 +62,25 @@ Verify Project Metadata END Close Browser +Verify Image Tag + [Arguments] ${json} ${verify_registry_name}=${false} + Log To Console "Verify Image Tag..." + @{project}= Get Value From Json ${json} $.projects.[*].name + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + FOR ${project} IN @{project} + @{registry_name}= Get Value From Json ${json} $.projects[?(@.name=${project})].registry_name + Run Keyword If '${registry_name}[0]' != '${null}' and '${verify_registry_name}' == '${false}' Continue For Loop + + @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image + ${has_image} Set Variable If ${out_has_image}[0] == ${true} ${true} ${false} + Go Into Project ${project} has_image=${has_image} + @{repo}= Get Value From Json ${json} $.projects[?(@.name=${project})]..repo..name + Run Keyword If ${has_image} == ${true} Loop Image Repo @{repo} + Navigate To Projects + END + Close Browser + Verify Checkbox [Arguments] ${json} ${key} ${checkbox} ${is_opposite}=${false} @{out}= Get Value From Json ${json} ${key} @@ -87,12 +96,15 @@ Loop Image Repo END Verify Member Exist - [Arguments] ${json} + [Arguments] ${json} ${verify_registry_name}=${false} Log To Console "Verify Member Exist..." @{project}= Get Value From Json ${json} $.projects.[*].name Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} FOR ${project} IN @{project} + @{registry_name}= Get Value From Json ${json} $.projects[?(@.name=${project})].registry_name + Run Keyword If '${registry_name}[0]' != '${null}' and '${verify_registry_name}' == '${false}' Continue For Loop + @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image ${has_image} Set Variable If ${out_has_image}[0] == ${true} ${true} ${false} Go Into Project ${project} has_image=${has_image} @@ -104,12 +116,15 @@ Verify Member Exist Close Browser Verify Webhook - [Arguments] ${json} + [Arguments] ${json} ${verify_registry_name}=${false} Log To Console "Verify Webhook..." @{project}= Get Value From Json ${json} $.projects.[*].name Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} FOR ${project} IN @{project} + @{registry_name}= Get Value From Json ${json} $.projects[?(@.name=${project})].registry_name + Run Keyword If '${registry_name}[0]' != '${null}' and '${verify_registry_name}' == '${false}' Continue For Loop + @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image ${has_image} Set Variable If ${out_has_image}[0] == ${true} ${true} ${false} Go Into Project ${project} has_image=${has_image} @@ -131,12 +146,15 @@ Verify Webhook Close Browser Verify Webhook For 2.0 - [Arguments] ${json} + [Arguments] ${json} ${verify_registry_name}=${false} Log To Console "Verify Webhook..." @{project}= Get Value From Json ${json} $.projects.[*].name Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} FOR ${project} IN @{project} + @{registry_name}= Get Value From Json ${json} $.projects[?(@.name=${project})].registry_name + Run Keyword If '${registry_name}[0]' != '${null}' and '${verify_registry_name}' == '${false}' Continue For Loop + @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image ${has_image} Set Variable If ${out_has_image}[0] == ${true} ${true} ${false} Go Into Project ${project} has_image=${has_image} @@ -162,12 +180,15 @@ Verify Webhook For 2.0 Close Browser Verify Tag Retention Rule - [Arguments] ${json} + [Arguments] ${json} ${verify_registry_name}=${false} Log To Console "Verify Tag Retention Rule..." @{project}= Get Value From Json ${json} $.projects.[*].name Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} FOR ${project} IN @{project} + @{registry_name}= Get Value From Json ${json} $.projects[?(@.name=${project})].registry_name + Run Keyword If '${registry_name}[0]' != '${null}' and '${verify_registry_name}' == '${false}' Continue For Loop + ${tag_retention_rule}= Get Value From Json ${json} $.projects[?(@.name=${project})].tag_retention_rule Run Keyword If ${tag_retention_rule}[0] == ${null} Continue For Loop ${out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image @@ -189,12 +210,15 @@ Verify Tag Retention Rule Close Browser Verify Tag Immutability Rule - [Arguments] ${json} + [Arguments] ${json} ${verify_registry_name}=${false} Log To Console "Verify Tag Immutability Rule..." @{project}= Get Value From Json ${json} $.projects.[*].name Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} FOR ${project} IN @{project} + @{registry_name}= Get Value From Json ${json} $.projects[?(@.name=${project})].registry_name + Run Keyword If '${registry_name}[0]' != '${null}' and '${verify_registry_name}' == '${false}' Continue For Loop + @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image ${has_image} Set Variable If ${out_has_image}[0] == ${true} ${true} ${false} Go Into Project ${project} has_image=${has_image} @@ -219,12 +243,15 @@ Loop Member END Verify Robot Account Exist - [Arguments] ${json} + [Arguments] ${json} ${verify_registry_name}=${false} Log To Console "Verify Robot Account Exist..." @{project}= Get Value From Json ${json} $.projects.[*].name Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} FOR ${project} IN @{project} + @{registry_name}= Get Value From Json ${json} $.projects[?(@.name=${project})].registry_name + Run Keyword If '${registry_name}[0]' != '${null}' and '${verify_registry_name}' == '${false}' Continue For Loop + @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image ${has_image} Set Variable If ${out_has_image}[0] == ${true} ${true} ${false} Go Into Project ${project} has_image=${has_image} @@ -267,12 +294,15 @@ Verify System Label Close Browser Verify Project Label - [Arguments] ${json} + [Arguments] ${json} ${verify_registry_name}=${false} Log To Console "Verify Project Label..." - @{project}= Get Value From Json ${json} $.peoject.[*].name + @{project}= Get Value From Json ${json} $.projects.[*].name Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} FOR ${project} IN @{project} + @{registry_name}= Get Value From Json ${json} $.projects[?(@.name=${project})].registry_name + Run Keyword If '${registry_name}[0]' != '${null}' and '${verify_registry_name}' == '${false}' Continue For Loop + @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image ${has_image} Set Variable If ${out_has_image}[0] == ${true} ${true} ${false} Go Into Project ${project} has_image=${has_image} @@ -381,12 +411,15 @@ Verify System Setting Close Browser Verify Project-level Allowlist - [Arguments] ${json} + [Arguments] ${json} ${verify_registry_name}=${false} Log To Console "Verify Project-level Allowlist..." @{project}= Get Value From Json ${json} $.projects.[*].name Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} FOR ${project} IN @{project} + @{registry_name}= Get Value From Json ${json} $.projects[?(@.name=${project})].registry_name + Run Keyword If '${registry_name}[0]' != '${null}' and '${verify_registry_name}' == '${false}' Continue For Loop + @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image ${has_image} Set Variable If ${out_has_image}[0] == ${true} ${true} ${false} Go Into Project ${project} has_image=${has_image} diff --git a/tests/robot-cases/Group3-Upgrade/data.json b/tests/robot-cases/Group3-Upgrade/data.json index 287dc8a9e..511bd4b2d 100644 --- a/tests/robot-cases/Group3-Upgrade/data.json +++ b/tests/robot-cases/Group3-Upgrade/data.json @@ -514,13 +514,13 @@ "auth_header":"bbb", "enabled":true, "notify_type":"slack", - "name":"webhook28" + "name":"webhook3.3" }, "configuration":{ "public":"true", - "enable_content_trust":"true", + "enable_content_trust":"false", "auto_scan":"true", - "prevent_vul":"true", + "prevent_vul":"false", "severity":"high", "reuse_sys_cve_allowlist":"false", "deployment_security":{ diff --git a/tests/robot-cases/Group3-Upgrade/feature_map.json b/tests/robot-cases/Group3-Upgrade/feature_map.json index 70a04de0d..dfd0c8867 100644 --- a/tests/robot-cases/Group3-Upgrade/feature_map.json +++ b/tests/robot-cases/Group3-Upgrade/feature_map.json @@ -1,456 +1,494 @@ -{ - "create_project":[ - { - "branch":1, - "version":"1.4" - }, - { - "branch":1, - "version":"1.6" - }, - { - "branch":1, - "version":"1.7" - }, - { - "branch":1, - "version":"1.8" - }, - { - "branch":2, - "version":"1.9" - }, - { - "branch":2, - "version":"1.10" - }, - { - "branch":2, - "version":"2.0" - }, - { - "branch":3, - "version":"2.1" - }, - { - "branch":3, - "version":"2.2" - } - ], - "add_member":[ - { - "branch":1, - "version":"1.4" - }, - { - "branch":2, - "version":"1.6" - }, - { - "branch":2, - "version":"1.7" - }, - { - "branch":2, - "version":"1.8" - }, - { - "branch":2, - "version":"1.9" - }, - { - "branch":2, - "version":"1.10" - }, - { - "branch":2, - "version":"2.0" - }, - { - "branch":2, - "version":"2.1" - }, - { - "branch":2, - "version":"2.2" - } - ], - "set_user_admin":[ - { - "branch":1, - "version":"1.4" - }, - { - "branch":2, - "version":"1.6" - }, - { - "branch":2, - "version":"1.7" - }, - { - "branch":2, - "version":"1.8" - }, - { - "branch":2, - "version":"1.9" - }, - { - "branch":2, - "version":"1.10" - }, - { - "branch":3, - "version":"2.0" - }, - { - "branch":3, - "version":"2.1" - }, - { - "branch":3, - "version":"2.2" - } - ], - "add_endpoint":[ - { - "branch":1, - "version":"1.4" - }, - { - "branch":1, - "version":"1.6" - }, - { - "branch":1, - "version":"1.7" - }, - { - "branch":2, - "version":"1.8" - }, - { - "branch":2, - "version":"1.9" - }, - { - "branch":2, - "version":"1.10" - }, - { - "branch":2, - "version":"2.0" - }, - { - "branch":2, - "version":"2.1" - }, - { - "branch":2, - "version":"2.2" - } - ], - "add_replication_rule":[ - { - "branch":1, - "version":"1.4" - }, - { - "branch":1, - "version":"1.6" - }, - { - "branch":1, - "version":"1.7" - }, - { - "branch":2, - "version":"1.8" - }, - { - "branch":2, - "version":"1.9" - }, - { - "branch":2, - "version":"1.10" - }, - { - "branch":2, - "version":"2.0" - }, - { - "branch":2, - "version":"2.1" - }, - { - "branch":2, - "version":"2.2" - } - ], - "add_sys_allowlist":[ - { - "branch":1, - "version":"1.9" - }, - { - "branch":1, - "version":"1.10" - }, - { - "branch":1, - "version":"2.0" - }, - { - "branch":2, - "version":"2.1" - }, - { - "branch":2, - "version":"2.2" - } - ], - "update_project_setting_allowlist":[ - { - "branch":1, - "version":"1.9" - }, - { - "branch":1, - "version":"1.10" - }, - { - "branch":1, - "version":"2.0" - }, - { - "branch":2, - "version":"2.1" - }, - { - "branch":2, - "version":"2.2" - } - ], - "add_project_robot_account":[ - { - "branch":1, - "version":"1.8" - }, - { - "branch":1, - "version":"1.9" - }, - { - "branch":1, - "version":"1.10" - }, - { - "branch":1, - "version":"2.0" - }, - { - "branch":1, - "version":"2.1" - }, - { - "branch":1, - "version":"2.2" - } - ], - "add_tag_retention_rule":[ - { - "branch":1, - "version":"1.9" - }, - { - "branch":1, - "version":"1.10" - }, - { - "branch":1, - "version":"2.0" - }, - { - "branch":1, - "version":"2.1" - }, - { - "branch":1, - "version":"2.2" - } - ], - "add_tag_immutability_rule":[ - { - "branch":1, - "version":"1.10" - }, - { - "branch":1, - "version":"2.0" - }, - { - "branch":1, - "version":"2.1" - }, - { - "branch":1, - "version":"2.2" - } - ], - "add_webhook":[ - { - "branch":1, - "version":"1.9" - }, - { - "branch":1, - "version":"1.10" - }, - { - "branch":2, - "version":"2.0" - }, - { - "branch":2, - "version":"2.1" - }, - { - "branch":2, - "version":"2.2" - } - ], - "update_interrogation_services":[ - { - "branch":1, - "version":"1.9" - }, - { - "branch":1, - "version":"1.10" - }, - { - "branch":1, - "version":"2.0" - }, - { - "branch":1, - "version":"2.1" - }, - { - "branch":1, - "version":"2.2" - } - ], - "push_artifact_index":[ - { - "branch":1, - "version":"2.0" - }, - { - "branch":1, - "version":"2.1" - }, - { - "branch":1, - "version":"2.2" - } - ], - "add_distribution":[ - { - "branch":1, - "version":"2.1" - }, - { - "branch":1, - "version":"2.2" - } - ], - "add_p2p_preheat_policy":[ - { - "branch":1, - "version":"2.1" - }, - { - "branch":1, - "version":"2.2" - } - ], - "get_ca":[ - { - "branch":1, - "version":"1.4" - }, - { - "branch":1, - "version":"1.6" - }, - { - "branch":1, - "version":"1.7" - }, - { - "branch":1, - "version":"1.8" - }, - { - "branch":1, - "version":"1.9" - }, - { - "branch":1, - "version":"1.10" - }, - { - "branch":2, - "version":"2.0" - }, - { - "branch":2, - "version":"2.1" - }, - { - "branch":2, - "version":"2.2" - } - ], - "set_url":[ - { - "branch":1, - "version":"1.4" - }, - { - "branch":1, - "version":"1.6" - }, - { - "branch":1, - "version":"1.7" - }, - { - "branch":1, - "version":"1.8" - }, - { - "branch":1, - "version":"1.9" - }, - { - "branch":1, - "version":"1.10" - }, - { - "branch":2, - "version":"2.0" - }, - { - "branch":2, - "version":"2.1" - }, - { - "branch":2, - "version":"2.2" - } - ] - } +{ + "create_project":[ + { + "branch":1, + "version":"1.4" + }, + { + "branch":1, + "version":"1.6" + }, + { + "branch":1, + "version":"1.7" + }, + { + "branch":1, + "version":"1.8" + }, + { + "branch":2, + "version":"1.9" + }, + { + "branch":2, + "version":"1.10" + }, + { + "branch":2, + "version":"2.0" + }, + { + "branch":3, + "version":"2.1" + }, + { + "branch":3, + "version":"2.2" + } + ], + "populate_projects":[ + { + "branch":1, + "version":"1.4" + }, + { + "branch":1, + "version":"1.6" + }, + { + "branch":1, + "version":"1.7" + }, + { + "branch":1, + "version":"1.8" + }, + { + "branch":2, + "version":"1.9" + }, + { + "branch":2, + "version":"1.10" + }, + { + "branch":2, + "version":"2.0" + }, + { + "branch":3, + "version":"2.1" + }, + { + "branch":3, + "version":"2.2" + } + ], + "add_member":[ + { + "branch":1, + "version":"1.4" + }, + { + "branch":2, + "version":"1.6" + }, + { + "branch":2, + "version":"1.7" + }, + { + "branch":2, + "version":"1.8" + }, + { + "branch":2, + "version":"1.9" + }, + { + "branch":2, + "version":"1.10" + }, + { + "branch":2, + "version":"2.0" + }, + { + "branch":2, + "version":"2.1" + }, + { + "branch":2, + "version":"2.2" + } + ], + "set_user_admin":[ + { + "branch":1, + "version":"1.4" + }, + { + "branch":2, + "version":"1.6" + }, + { + "branch":2, + "version":"1.7" + }, + { + "branch":2, + "version":"1.8" + }, + { + "branch":2, + "version":"1.9" + }, + { + "branch":2, + "version":"1.10" + }, + { + "branch":3, + "version":"2.0" + }, + { + "branch":3, + "version":"2.1" + }, + { + "branch":3, + "version":"2.2" + } + ], + "add_endpoint":[ + { + "branch":1, + "version":"1.4" + }, + { + "branch":1, + "version":"1.6" + }, + { + "branch":1, + "version":"1.7" + }, + { + "branch":2, + "version":"1.8" + }, + { + "branch":2, + "version":"1.9" + }, + { + "branch":2, + "version":"1.10" + }, + { + "branch":2, + "version":"2.0" + }, + { + "branch":2, + "version":"2.1" + }, + { + "branch":2, + "version":"2.2" + } + ], + "add_replication_rule":[ + { + "branch":1, + "version":"1.4" + }, + { + "branch":1, + "version":"1.6" + }, + { + "branch":1, + "version":"1.7" + }, + { + "branch":2, + "version":"1.8" + }, + { + "branch":2, + "version":"1.9" + }, + { + "branch":2, + "version":"1.10" + }, + { + "branch":2, + "version":"2.0" + }, + { + "branch":2, + "version":"2.1" + }, + { + "branch":2, + "version":"2.2" + } + ], + "add_sys_allowlist":[ + { + "branch":1, + "version":"1.9" + }, + { + "branch":1, + "version":"1.10" + }, + { + "branch":1, + "version":"2.0" + }, + { + "branch":2, + "version":"2.1" + }, + { + "branch":2, + "version":"2.2" + } + ], + "update_project_setting_allowlist":[ + { + "branch":1, + "version":"1.9" + }, + { + "branch":1, + "version":"1.10" + }, + { + "branch":1, + "version":"2.0" + }, + { + "branch":2, + "version":"2.1" + }, + { + "branch":2, + "version":"2.2" + } + ], + "add_project_robot_account":[ + { + "branch":1, + "version":"1.8" + }, + { + "branch":1, + "version":"1.9" + }, + { + "branch":1, + "version":"1.10" + }, + { + "branch":1, + "version":"2.0" + }, + { + "branch":1, + "version":"2.1" + }, + { + "branch":1, + "version":"2.2" + } + ], + "add_tag_retention_rule":[ + { + "branch":1, + "version":"1.9" + }, + { + "branch":1, + "version":"1.10" + }, + { + "branch":1, + "version":"2.0" + }, + { + "branch":1, + "version":"2.1" + }, + { + "branch":1, + "version":"2.2" + } + ], + "add_tag_immutability_rule":[ + { + "branch":1, + "version":"1.10" + }, + { + "branch":1, + "version":"2.0" + }, + { + "branch":1, + "version":"2.1" + }, + { + "branch":1, + "version":"2.2" + } + ], + "add_webhook":[ + { + "branch":1, + "version":"1.9" + }, + { + "branch":1, + "version":"1.10" + }, + { + "branch":2, + "version":"2.0" + }, + { + "branch":2, + "version":"2.1" + }, + { + "branch":2, + "version":"2.2" + } + ], + "update_interrogation_services":[ + { + "branch":1, + "version":"1.9" + }, + { + "branch":1, + "version":"1.10" + }, + { + "branch":1, + "version":"2.0" + }, + { + "branch":1, + "version":"2.1" + }, + { + "branch":1, + "version":"2.2" + } + ], + "push_artifact_index":[ + { + "branch":1, + "version":"2.0" + }, + { + "branch":1, + "version":"2.1" + }, + { + "branch":1, + "version":"2.2" + } + ], + "add_distribution":[ + { + "branch":1, + "version":"2.1" + }, + { + "branch":1, + "version":"2.2" + } + ], + "add_p2p_preheat_policy":[ + { + "branch":1, + "version":"2.1" + }, + { + "branch":1, + "version":"2.2" + } + ], + "get_ca":[ + { + "branch":1, + "version":"1.4" + }, + { + "branch":1, + "version":"1.6" + }, + { + "branch":1, + "version":"1.7" + }, + { + "branch":1, + "version":"1.8" + }, + { + "branch":1, + "version":"1.9" + }, + { + "branch":1, + "version":"1.10" + }, + { + "branch":2, + "version":"2.0" + }, + { + "branch":2, + "version":"2.1" + }, + { + "branch":2, + "version":"2.2" + } + ], + "set_url":[ + { + "branch":1, + "version":"1.4" + }, + { + "branch":1, + "version":"1.6" + }, + { + "branch":1, + "version":"1.7" + }, + { + "branch":1, + "version":"1.8" + }, + { + "branch":1, + "version":"1.9" + }, + { + "branch":1, + "version":"1.10" + }, + { + "branch":2, + "version":"2.0" + }, + { + "branch":2, + "version":"2.1" + }, + { + "branch":2, + "version":"2.2" + } + ] + } diff --git a/tests/robot-cases/Group3-Upgrade/prepare.py b/tests/robot-cases/Group3-Upgrade/prepare.py index 4ffca5a55..3e0257e42 100644 --- a/tests/robot-cases/Group3-Upgrade/prepare.py +++ b/tests/robot-cases/Group3-Upgrade/prepare.py @@ -55,20 +55,58 @@ def get_feature_branch(func): return inner_func class HarborAPI: + @get_feature_branch + def populate_projects(self, **kwargs): + for project in data["projects"]: + if kwargs["branch"] == 1: + if project["registry_name"] is not None: + continue + elif kwargs["branch"] == 2: + if project["registry_name"] is not None: + continue + elif kwargs["branch"] == 3: + print("Populate all projects") + else: + raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch)) + self.create_project(project, version=args.version) + for member in project["member"]: + self.add_member(project["name"], member["name"], member["role"], version=args.version) + for robot_account in project["robot_account"]: + self.add_project_robot_account(project["name"], robot_account, version=args.version) + self.add_p2p_preheat_policy(project, version=args.version) + self.add_webhook(project["name"], project["webhook"], version=args.version) + if project["tag_retention_rule"] is not None: + self.add_tag_retention_rule(project["name"], project["tag_retention_rule"], version=args.version) + self.add_tag_immutability_rule(project["name"], project["tag_immutability_rule"], version=args.version) + self.update_project_setting_metadata(project["name"], + project["configuration"]["public"], + project["configuration"]["enable_content_trust"], + project["configuration"]["prevent_vul"], + project["configuration"]["severity"], + project["configuration"]["auto_scan"]) + self.update_project_setting_allowlist(project["name"], + project["configuration"]["reuse_sys_cve_allowlist"], + project["configuration"]["deployment_security"], version=args.version) + time.sleep(30) + @get_feature_branch def create_project(self, project, **kwargs): if kwargs["branch"] == 1: - body=dict(body={"project_name": project["name"], "metadata": {"public": "true"}}) + body=dict(body={"project_name": project["name"], "metadata": {"public": "true"}}) + request(url+"projects", 'post', **body) elif kwargs["branch"] == 2: - body=dict(body={"project_name": project["name"], "metadata": {"public": "true"},"count_limit":project["count_limit"],"storage_limit":project["storage_limit"]}) + body=dict(body={"project_name": project["name"], "metadata": {"public": "true"},"count_limit":project["count_limit"],"storage_limit":project["storage_limit"]}) + request(url+"projects", 'post', **body) elif kwargs["branch"] == 3: if project["registry_name"] is not None: r = request(url+"registries?name="+project["registry_name"]+"", 'get') registry_id = int(str(r.json()[0]['id'])) else: - registry_id=None + registry_id = None body=dict(body={"project_name": project["name"], "registry_id":registry_id, "metadata": {"public": "true"},"storage_limit":project["storage_limit"]}) request(url+"projects", 'post', **body) + + #Project with registry_name must have repo and to verify repo can be pulled. if project["registry_name"] is not None: USER_ADMIN=dict(endpoint = "https://"+args.endpoint+"/api/v2.0" , username = "admin", password = "Harbor12345") repo = Repository() @@ -77,11 +115,9 @@ class HarborAPI: time.sleep(180) repo_name = urllib.parse.quote(_repo["cache_image_namespace"]+"/"+_repo["cache_image"],'utf-8') repo_data = repo.get_repository(project["name"], repo_name, **USER_ADMIN) - print("=========repo_data:",repo_data) return else: raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch)) - request(url+"projects", 'post', **body) def create_user(self, username): payload = {"username":username, "email":username+"@vmware.com", "password":"Harbor12345", "realname":username, "comment":"string"} @@ -546,7 +582,7 @@ class HarborAPI: repo_name_b, tag_b = push_image_to_project(project, args.endpoint, 'admin', 'Harbor12345', image_b, "latest") manifests = [args.endpoint+"/"+repo_name_a+":"+tag_a, args.endpoint+"/"+repo_name_b+":"+tag_b] index = args.endpoint+"/"+project+"/"+name+":"+tag - docker_manifest_push_to_harbor(index, manifests, args.endpoint, 'admin', 'Harbor12345', cfg_file = args.libpath + "/update_docker_cfg.sh" + docker_manifest_push_to_harbor(index, manifests, args.endpoint, 'admin', 'Harbor12345', cfg_file = args.libpath + "/update_docker_cfg.sh") def request(url, method, user = None, userp = None, **kwargs): if user is None: @@ -607,18 +643,7 @@ def do_data_creation(): for distribution in data["distributions"]: harborAPI.add_distribution(distribution, version=args.version) - for project in data["projects"]: - harborAPI.create_project(project, version=args.version) - for member in project["member"]: - harborAPI.add_member(project["name"], member["name"], member["role"], version=args.version) - for robot_account in project["robot_account"]: - harborAPI.add_project_robot_account(project["name"], robot_account, version=args.version) - harborAPI.add_p2p_preheat_policy(project, version=args.version) - harborAPI.add_webhook(project["name"], project["webhook"], version=args.version) - if project["tag_retention_rule"] in not None: - harborAPI.add_tag_retention_rule(project["name"], project["tag_retention_rule"], version=args.version) - harborAPI.add_tag_immutability_rule(project["name"], project["tag_immutability_rule"], version=args.version) - time.sleep(30) + harborAPI.populate_projects(version=args.version) harborAPI.push_artifact_index(data["projects"][0]["name"], data["projects"][0]["artifact_index"]["name"], data["projects"][0]["artifact_index"]["tag"], version=args.version) pull_image("busybox", "redis", "haproxy", "alpine", "httpd:2") @@ -628,18 +653,6 @@ def do_data_creation(): for replicationrule in data["replicationrule"]: harborAPI.add_replication_rule(replicationrule, version=args.version) - for project in data["projects"]: - harborAPI.update_project_setting_metadata(project["name"], - project["configuration"]["public"], - project["configuration"]["enable_content_trust"], - project["configuration"]["prevent_vul"], - project["configuration"]["severity"], - project["configuration"]["auto_scan"]) - - for project in data["projects"]: - harborAPI.update_project_setting_allowlist(project["name"], - project["configuration"]["reuse_sys_cve_allowlist"], - project["configuration"]["deployment_security"], version=args.version) harborAPI.update_interrogation_services(data["interrogation_services"]["cron"], version=args.version) diff --git a/tests/robot-cases/Group3-Upgrade/verify.robot b/tests/robot-cases/Group3-Upgrade/verify.robot index 8d6272ef1..b09253df2 100644 --- a/tests/robot-cases/Group3-Upgrade/verify.robot +++ b/tests/robot-cases/Group3-Upgrade/verify.robot @@ -1,106 +1,116 @@ -*** Settings *** -Documentation Harbor BATs -Resource ../../resources/Util.robot -Default Tags Nightly - -*** Variables *** -${HARBOR_URL} https://${ip} -${HARBOR_ADMIN} admin - -*** Test Cases *** -Test Case - Upgrade Verify - [Tags] 1.8-latest - ${data}= Load Json From File ${CURDIR}${/}data.json - Run Keyword Verify User ${data} - Run Keyword Verify Project ${data} - Run Keyword Verify Member Exist ${data} - Run Keyword Verify Robot Account Exist ${data} - Run Keyword Verify User System Admin Role ${data} - Run Keyword Verify Endpoint ${data} - Run Keyword Verify Replicationrule ${data} - Run Keyword Verify System Setting ${data} - Run Keyword Verify Image Tag ${data} - -Test Case - Upgrade Verify - [Tags] 1.9-latest - ${data}= Load Json From File ${CURDIR}${/}data.json - Run Keyword Verify User ${data} - Run Keyword Verify Project ${data} - Run Keyword Verify Member Exist ${data} - Run Keyword Verify Robot Account Exist ${data} - Run Keyword Verify Project-level Allowlist ${data} - Run Keyword Verify Webhook ${data} - Run Keyword Verify Tag Retention Rule ${data} - Run Keyword Verify User System Admin Role ${data} - Run Keyword Verify Endpoint ${data} - Run Keyword Verify Replicationrule ${data} - Run Keyword Verify Interrogation Services ${data} - Run Keyword Verify System Setting ${data} - Run Keyword Verify System Setting Allowlist ${data} - Run Keyword Verify Image Tag ${data} - Run Keyword Verify Trivy Is Default Scanner - -Test Case - Upgrade Verify - [Tags] 1.10-latest - ${data}= Load Json From File ${CURDIR}${/}data.json - Run Keyword Verify User ${data} - Run Keyword Verify Project ${data} - Run Keyword Verify Member Exist ${data} - Run Keyword Verify Robot Account Exist ${data} - Run Keyword Verify Project-level Allowlist ${data} - Run Keyword Verify Webhook ${data} - Run Keyword Verify Tag Retention Rule ${data} - Run Keyword Verify Tag Immutability Rule ${data} - Run Keyword Verify User System Admin Role ${data} - Run Keyword Verify Endpoint ${data} - Run Keyword Verify Replicationrule ${data} - Run Keyword Verify Interrogation Services ${data} - Run Keyword Verify System Setting ${data} - Run Keyword Verify System Setting Allowlist ${data} - Run Keyword Verify Image Tag ${data} - Run Keyword Verify Clair Is Default Scanner - -Test Case - Upgrade Verify - [Tags] 2.0-latest - ${data}= Load Json From File ${CURDIR}${/}data.json - Run Keyword Verify User ${data} - Run Keyword Verify Project ${data} - Run Keyword Verify Member Exist ${data} - Run Keyword Verify Robot Account Exist ${data} - Run Keyword Verify Project-level Allowlist ${data} - Run Keyword Verify Webhook For 2.0 ${data} - Run Keyword Verify Tag Retention Rule ${data} - Run Keyword Verify Tag Immutability Rule ${data} - Run Keyword Verify User System Admin Role ${data} - Run Keyword Verify Endpoint ${data} - Run Keyword Verify Replicationrule ${data} - Run Keyword Verify Interrogation Services ${data} - Run Keyword Verify System Setting ${data} - Run Keyword Verify System Setting Allowlist ${data} - Run Keyword Verify Image Tag ${data} - Run Keyword Verify Trivy Is Default Scanner - Run Keyword Verify Artifact Index ${data} - -Test Case - Upgrade Verify - [Tags] 2.1-latest - ${data}= Load Json From File ${CURDIR}${/}data.json - Run Keyword Verify User ${data} - Run Keyword Verify Project ${data} check_content_trust=${false} - Run Keyword Verify Member Exist ${data} - Run Keyword Verify Robot Account Exist ${data} - Run Keyword Verify Project-level Allowlist ${data} - Run Keyword Verify Webhook For 2.0 ${data} - Run Keyword Verify Tag Retention Rule ${data} - Run Keyword Verify Tag Immutability Rule ${data} - Run Keyword Verify User System Admin Role ${data} - Run Keyword Verify Endpoint ${data} - Run Keyword Verify Replicationrule ${data} - Run Keyword Verify Interrogation Services ${data} - Run Keyword Verify System Setting ${data} - Run Keyword Verify System Setting Allowlist ${data} - Run Keyword Verify Image Tag ${data} - Run Keyword Verify Trivy Is Default Scanner - Run Keyword Verify Artifact Index ${data} - Run Keyword Verify Proxy Cache Image Existence ${data} - Run Keyword Verify Distributions ${data} - Run Keyword Verify P2P Preheat Policy ${data} +*** Settings *** +Documentation Harbor BATs +Resource ../../resources/Util.robot +Default Tags Nightly + +*** Variables *** +${HARBOR_URL} https://${ip} +${HARBOR_ADMIN} admin + +*** Test Cases *** +Test Case - Upgrade Verify + [Tags] 1.8-latest + ${data}= Load Json From File ${CURDIR}${/}data.json + Run Keyword Verify User ${data} + Run Keyword Verify Project ${data} + Run Keyword Verify Project Label ${data} + Run Keyword Verify Project Metadata ${data} + Run Keyword Verify Member Exist ${data} + Run Keyword Verify Robot Account Exist ${data} + Run Keyword Verify User System Admin Role ${data} + Run Keyword Verify Endpoint ${data} + Run Keyword Verify Replicationrule ${data} + Run Keyword Verify System Setting ${data} + Run Keyword Verify Image Tag ${data} + +Test Case - Upgrade Verify + [Tags] 1.9-latest + ${data}= Load Json From File ${CURDIR}${/}data.json + Run Keyword Verify User ${data} + Run Keyword Verify Project ${data} + Run Keyword Verify Project Label ${data} + Run Keyword Verify Project Metadata ${data} + Run Keyword Verify Member Exist ${data} + Run Keyword Verify Robot Account Exist ${data} + Run Keyword Verify Project-level Allowlist ${data} + Run Keyword Verify Webhook ${data} + Run Keyword Verify Tag Retention Rule ${data} + Run Keyword Verify User System Admin Role ${data} + Run Keyword Verify Endpoint ${data} + Run Keyword Verify Replicationrule ${data} + Run Keyword Verify Interrogation Services ${data} + Run Keyword Verify System Setting ${data} + Run Keyword Verify System Setting Allowlist ${data} + Run Keyword Verify Image Tag ${data} + Run Keyword Verify Trivy Is Default Scanner + +Test Case - Upgrade Verify + [Tags] 1.10-latest + ${data}= Load Json From File ${CURDIR}${/}data.json + Run Keyword Verify User ${data} + Run Keyword Verify Project ${data} + Run Keyword Verify Project Label ${data} + Run Keyword Verify Project Metadata ${data} + Run Keyword Verify Member Exist ${data} + Run Keyword Verify Robot Account Exist ${data} + Run Keyword Verify Project-level Allowlist ${data} + Run Keyword Verify Webhook ${data} + Run Keyword Verify Tag Retention Rule ${data} + Run Keyword Verify Tag Immutability Rule ${data} + Run Keyword Verify User System Admin Role ${data} + Run Keyword Verify Endpoint ${data} + Run Keyword Verify Replicationrule ${data} + Run Keyword Verify Interrogation Services ${data} + Run Keyword Verify System Setting ${data} + Run Keyword Verify System Setting Allowlist ${data} + Run Keyword Verify Image Tag ${data} + Run Keyword Verify Clair Is Default Scanner + +Test Case - Upgrade Verify + [Tags] 2.0-latest + ${data}= Load Json From File ${CURDIR}${/}data.json + Run Keyword Verify User ${data} + Run Keyword Verify Project ${data} + Run Keyword Verify Project Metadata ${data} check_content_trust=${false} + #Run Keyword Verify Project Label ${data} + Run Keyword Verify Member Exist ${data} + Run Keyword Verify Robot Account Exist ${data} + Run Keyword Verify Project-level Allowlist ${data} + Run Keyword Verify Webhook For 2.0 ${data} + Run Keyword Verify Tag Retention Rule ${data} + Run Keyword Verify Tag Immutability Rule ${data} + Run Keyword Verify User System Admin Role ${data} + Run Keyword Verify Endpoint ${data} + Run Keyword Verify Replicationrule ${data} + Run Keyword Verify Interrogation Services ${data} + Run Keyword Verify System Setting ${data} + Run Keyword Verify System Setting Allowlist ${data} + Run Keyword Verify Image Tag ${data} + Run Keyword Verify Trivy Is Default Scanner + Run Keyword Verify Artifact Index ${data} + +Test Case - Upgrade Verify + [Tags] 2.1-latest + ${data}= Load Json From File ${CURDIR}${/}data.json + Run Keyword Verify User ${data} + Run Keyword Verify Project ${data} verify_registry_name=${true} + Run Keyword Verify Project Metadata ${data} check_content_trust=${false} verify_registry_name=${true} + #Run Keyword Verify Project Label ${data} verify_registry_name=${true} + Run Keyword Verify Member Exist ${data} verify_registry_name=${true} + Run Keyword Verify Robot Account Exist ${data} verify_registry_name=${true} + Run Keyword Verify Project-level Allowlist ${data} verify_registry_name=${true} + Run Keyword Verify Webhook For 2.0 ${data} verify_registry_name=${true} + Run Keyword Verify Tag Retention Rule ${data} verify_registry_name=${true} + Run Keyword Verify Tag Immutability Rule ${data} verify_registry_name=${true} + Run Keyword Verify Image Tag ${data} verify_registry_name=${true} + Run Keyword Verify User System Admin Role ${data} + Run Keyword Verify Endpoint ${data} + Run Keyword Verify Replicationrule ${data} + Run Keyword Verify Interrogation Services ${data} + Run Keyword Verify System Setting ${data} + Run Keyword Verify System Setting Allowlist ${data} + Run Keyword Verify Trivy Is Default Scanner + Run Keyword Verify Artifact Index ${data} + Run Keyword Verify Proxy Cache Image Existence ${data} + Run Keyword Verify Distributions ${data} + Run Keyword Verify P2P Preheat Policy ${data}